[release-0.19] Disable ETCD Learner Mode (aws#7719)

* Disable ETCD Learner Mode in v1.29 ETCD Learner Mode went to Beta in k8s 1.29 and is now default enabled. When a new stacked etcd instance comes up, it joins the cluster in learner mode. The API Server cannot perform rpc calls against its etcd instance, and fails to come online. In theory the etcd instance should be promoted to a full member and this issue should be resolved, but for some reason this is not happening. While we investigate a root cause, we are disabling this new feature flag explicitly. * Extract semver computation required for Kubernetes v1.29 checks --------- Co-authored-by: Jonathan Meier <[email protected]> Co-authored-by: Abhay Krishna Arunachalam <[email protected]>
jonathanmeier5 · Feb 27, 2024 · 97053a9 · 97053a9
1 parent 879b80f
commit 97053a9
Show file tree

Hide file tree

Showing 8 changed files with 49 additions and 11 deletions.
diff --git a/pkg/providers/cloudstack/config/template-cp.yaml b/pkg/providers/cloudstack/config/template-cp.yaml
@@ -67,6 +67,10 @@ spec:
       name: {{.controlPlaneTemplateName}}
   kubeadmConfigSpec:
     clusterConfiguration:
+{{- if (ge (atoi $kube_minor_version) 29) }}
+      featureGates:
+        EtcdLearnerMode: false
+{{- end }}
       imageRepository: {{.kubernetesRepository}}
       etcd:
 {{- if .externalEtcd }}

diff --git a/pkg/providers/cloudstack/testdata/expected_results_encryption_config_cp_1_29.yaml b/pkg/providers/cloudstack/testdata/expected_results_encryption_config_cp_1_29.yaml
@@ -59,6 +59,8 @@ spec:
       name: test-control-plane-template-1234567890000
   kubeadmConfigSpec:
     clusterConfiguration:
+      featureGates:
+        EtcdLearnerMode: false
       imageRepository: public.ecr.aws/eks-distro/kubernetes
       etcd:
         local:

diff --git a/pkg/providers/snow/apibuilder.go b/pkg/providers/snow/apibuilder.go
@@ -58,6 +58,16 @@ func KubeadmControlPlane(log logr.Logger, clusterSpec *cluster.Spec, snowMachine
 
 	machineConfig := clusterSpec.SnowMachineConfig(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name)
 
+	kubeVersionSemver, err := semver.New(string(clusterSpec.Cluster.Spec.KubernetesVersion) + ".0")
+	if err != nil {
+		return nil, fmt.Errorf("error converting kubeVersion %v to semver %v", clusterSpec.Cluster.Spec.KubernetesVersion, err)
+	}
+
+	kube129Semver, err := semver.New(string(v1alpha1.Kube129) + ".0")
+	if err != nil {
+		return nil, fmt.Errorf("error converting kubeVersion %v to semver %v", v1alpha1.Kube129, err)
+	}
+
 	osFamily := machineConfig.OSFamily()
 	switch osFamily {
 	case v1alpha1.Bottlerocket:
@@ -70,24 +80,20 @@ func KubeadmControlPlane(log logr.Logger, clusterSpec *cluster.Spec, snowMachine
 		addBottlerocketBootstrapSnowInKubeadmControlPlane(kcp, versionsBundle.Snow.BottlerocketBootstrapSnow)
 		clusterapi.SetBottlerocketHostConfigInKubeadmControlPlane(kcp, machineConfig.Spec.HostOSConfiguration)
 
+		if kubeVersionSemver.Compare(kube129Semver) != -1 {
+			disableEtcdLearnerMode(kcp)
+		}
+
 	case v1alpha1.Ubuntu:
 		kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands,
 			"/etc/eks/bootstrap.sh",
 		)
-		kubeVersionSemver, err := semver.New(string(clusterSpec.Cluster.Spec.KubernetesVersion) + ".0")
-		if err != nil {
-			return nil, fmt.Errorf("error converting kubeVersion %v to semver %v", clusterSpec.Cluster.Spec.KubernetesVersion, err)
-		}
-
-		kube129Semver, err := semver.New(string(v1alpha1.Kube129) + ".0")
-		if err != nil {
-			return nil, fmt.Errorf("error converting kubeVersion %v to semver %v", v1alpha1.Kube129, err)
-		}
 
 		if kubeVersionSemver.Compare(kube129Semver) != -1 {
 			kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands,
 				"if [ -f /run/kubeadm/kubeadm.yaml ]; then sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml; fi",
 			)
+			disableEtcdLearnerMode(kcp)
 		}
 
 		if err := clusterapi.SetProxyConfigInKubeadmControlPlaneForUbuntu(kcp, clusterSpec.Cluster); err != nil {

diff --git a/pkg/providers/snow/apibuilder_test.go b/pkg/providers/snow/apibuilder_test.go
@@ -108,7 +108,7 @@ func wantKubeadmControlPlane(kubeVersion v1alpha1.KubernetesVersion) *controlpla
 	wantReplicas := int32(3)
 	wantMaxSurge := intstr.FromInt(1)
 	versionBundles := givenVersionsBundle(kubeVersion)
-	return &controlplanev1.KubeadmControlPlane{
+	kcp := &controlplanev1.KubeadmControlPlane{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "controlplane.cluster.x-k8s.io/v1beta1",
 			Kind:       "KubeadmControlPlane",
@@ -200,6 +200,14 @@ func wantKubeadmControlPlane(kubeVersion v1alpha1.KubernetesVersion) *controlpla
 			Version:  versionBundles.KubeDistro.Kubernetes.Tag,
 		},
 	}
+
+	if kubeVersion == "1.29" {
+		kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.FeatureGates = map[string]bool{
+			"EtcdLearnerMode": false,
+		}
+	}
+
+	return kcp
 }
 
 func wantKubeadmControlPlaneUnstackedEtcd() *controlplanev1.KubeadmControlPlane {

diff --git a/pkg/providers/snow/etcd.go b/pkg/providers/snow/etcd.go
@@ -15,3 +15,11 @@ func addStackedEtcdExtraArgsInKubeadmControlPlane(kcp *controlplanev1.KubeadmCon
 	stackedEtcdExtraArgs["listen-peer-urls"] = "https://0.0.0.0:2380"
 	stackedEtcdExtraArgs["listen-client-urls"] = "https://0.0.0.0:2379"
 }
+
+func disableEtcdLearnerMode(kcp *controlplanev1.KubeadmControlPlane) {
+	if kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.FeatureGates == nil {
+		kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.FeatureGates = map[string]bool{}
+	}
+
+	kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.FeatureGates["EtcdLearnerMode"] = false
+}
diff --git a/pkg/providers/tinkerbell/config/template-cp.yaml b/pkg/providers/tinkerbell/config/template-cp.yaml
@@ -1,3 +1,4 @@
+{{- $kube_minor_version := (index (splitList "." (trimPrefix "v" .kubernetesVersion)) 1) -}}
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: Cluster
 metadata:
@@ -37,6 +38,10 @@ metadata:
 spec:
   kubeadmConfigSpec:
     clusterConfiguration:
+{{- if (ge (atoi $kube_minor_version) 29) }}
+      featureGates:
+        EtcdLearnerMode: false
+{{- end }}
       imageRepository: {{.kubernetesRepository}}
       etcd:
 {{- if .externalEtcd }}
@@ -395,7 +400,6 @@ spec:
       - {{ . }}
       {{- end }}
 {{- end }}
-{{- $kube_minor_version := (index (splitList "." (trimPrefix "v" .kubernetesVersion)) 1) }}
 {{- if and (or .registryMirrorMap .proxyConfig (ge (atoi $kube_minor_version) 29)) (ne .format "bottlerocket") }}
     preKubeadmCommands:
 {{- if .registryMirrorMap }}

diff --git a/pkg/providers/vsphere/config/template-cp.yaml b/pkg/providers/vsphere/config/template-cp.yaml
@@ -86,6 +86,10 @@ spec:
       name: {{.controlPlaneTemplateName}}
   kubeadmConfigSpec:
     clusterConfiguration:
+{{- if (ge (atoi $kube_minor_version) 29) }}
+      featureGates:
+        EtcdLearnerMode: false
+{{- end }}
       imageRepository: {{.kubernetesRepository}}
       etcd:
 {{- if .externalEtcd }}

diff --git a/pkg/providers/vsphere/testdata/expected_results_ubuntu_etcd_encryption_cp_1_29.yaml b/pkg/providers/vsphere/testdata/expected_results_ubuntu_etcd_encryption_cp_1_29.yaml
@@ -78,6 +78,8 @@ spec:
       name: test-control-plane-template-1234567890000
   kubeadmConfigSpec:
     clusterConfiguration:
+      featureGates:
+        EtcdLearnerMode: false
       imageRepository: public.ecr.aws/eks-distro/kubernetes
       etcd:
         external: