diff --git a/test/e2e/data/infrastructure-aws/withclusterclass/e2e_test_templates/cluster-template-nested-multitenancy-clusterclass.yaml b/test/e2e/data/infrastructure-aws/withclusterclass/e2e_test_templates/cluster-template-nested-multitenancy-clusterclass.yaml index 84bdf3606a..5195d99d84 100644 --- a/test/e2e/data/infrastructure-aws/withclusterclass/e2e_test_templates/cluster-template-nested-multitenancy-clusterclass.yaml +++ b/test/e2e/data/infrastructure-aws/withclusterclass/e2e_test_templates/cluster-template-nested-multitenancy-clusterclass.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external name: ${CLUSTER_NAME} spec: clusterNetwork: @@ -56,6 +58,32 @@ spec: name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSClusterRoleIdentity metadata: @@ -80,3 +108,987 @@ spec: sourceIdentityRef: kind: AWSClusterRoleIdentity name: ${MULTI_TENANCY_JUMP_IDENTITY_NAME} +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon diff --git a/test/e2e/data/infrastructure-aws/withclusterclass/e2e_test_templates/cluster-template-self-hosted-clusterclass.yaml b/test/e2e/data/infrastructure-aws/withclusterclass/e2e_test_templates/cluster-template-self-hosted-clusterclass.yaml index 5c1054bf5b..4483a6cdfe 100644 --- a/test/e2e/data/infrastructure-aws/withclusterclass/e2e_test_templates/cluster-template-self-hosted-clusterclass.yaml +++ b/test/e2e/data/infrastructure-aws/withclusterclass/e2e_test_templates/cluster-template-self-hosted-clusterclass.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external name: ${CLUSTER_NAME} spec: clusterNetwork: @@ -55,3 +57,1013 @@ spec: - kind: ConfigMap name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon diff --git a/test/e2e/data/infrastructure-aws/withclusterclass/e2e_test_templates/cluster-template-topology.yaml b/test/e2e/data/infrastructure-aws/withclusterclass/e2e_test_templates/cluster-template-topology.yaml index 8f4f1bc1d6..ee269b8017 100644 --- a/test/e2e/data/infrastructure-aws/withclusterclass/e2e_test_templates/cluster-template-topology.yaml +++ b/test/e2e/data/infrastructure-aws/withclusterclass/e2e_test_templates/cluster-template-topology.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external name: ${CLUSTER_NAME} spec: clusterNetwork: @@ -51,3 +53,1013 @@ spec: - kind: ConfigMap name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-csimigration-off.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-csimigration-off.yaml index 29bcd7bfb6..ac3f822c58 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-csimigration-off.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-csimigration-off.yaml @@ -23,9 +23,6 @@ kind: AWSCluster metadata: name: ${CLUSTER_NAME} spec: - network: - vpc: - availabilityZoneUsageLimit: 1 region: ${AWS_REGION} sshKeyName: ${AWS_SSH_KEY_NAME} --- diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-efs-support.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-efs-support.yaml index 60e8c67907..e299b8d647 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-efs-support.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-efs-support.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 csi: external name: ${CLUSTER_NAME} @@ -39,19 +40,19 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-provider: aws + cloud-provider: external controllerManager: extraArgs: - cloud-provider: aws + cloud-provider: external initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' machineTemplate: infrastructureRef: @@ -118,7 +119,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' --- apiVersion: v1 @@ -142,6 +143,19 @@ spec: --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet metadata: name: crs-csi spec: @@ -154,6 +168,193 @@ spec: strategy: ApplyOnce --- apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 data: aws-efs-csi-external.yaml: |- apiVersion: v1 diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-external-csi.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-external-csi.yaml index 74c7d8b6b8..0e56deb3f6 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-external-csi.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-external-csi.yaml @@ -24,9 +24,6 @@ kind: AWSCluster metadata: name: ${CLUSTER_NAME} spec: - network: - vpc: - availabilityZoneUsageLimit: 1 region: ${AWS_REGION} sshKeyName: ${AWS_SSH_KEY_NAME} --- @@ -153,7 +150,7 @@ spec: --- apiVersion: v1 data: - aws-ebs-csi-external.yaml: |- + aws-ebs-csi-external.yaml: | apiVersion: v1 kind: Secret metadata: @@ -181,13 +178,28 @@ data: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-external-attacher-role + rules: rules: - apiGroups: - - "" + - '' resources: - persistentvolumes verbs: @@ -197,7 +209,7 @@ data: - update - patch - apiGroups: - - "" + - '' resources: - nodes verbs: @@ -237,7 +249,7 @@ data: name: ebs-external-provisioner-role rules: - apiGroups: - - "" + - '' resources: - persistentvolumes verbs: @@ -247,7 +259,7 @@ data: - create - delete - apiGroups: - - "" + - '' resources: - persistentvolumeclaims verbs: @@ -264,7 +276,7 @@ data: - list - watch - apiGroups: - - "" + - '' resources: - events verbs: @@ -296,7 +308,7 @@ data: - list - watch - apiGroups: - - "" + - '' resources: - nodes verbs: @@ -331,7 +343,7 @@ data: name: ebs-external-resizer-role rules: - apiGroups: - - "" + - '' resources: - persistentvolumes verbs: @@ -341,7 +353,7 @@ data: - update - patch - apiGroups: - - "" + - '' resources: - persistentvolumeclaims verbs: @@ -349,7 +361,7 @@ data: - list - watch - apiGroups: - - "" + - '' resources: - persistentvolumeclaims/status verbs: @@ -364,7 +376,7 @@ data: - list - watch - apiGroups: - - "" + - '' resources: - events verbs: @@ -374,7 +386,7 @@ data: - update - patch - apiGroups: - - "" + - '' resources: - pods verbs: @@ -390,7 +402,7 @@ data: name: ebs-external-snapshotter-role rules: - apiGroups: - - "" + - '' resources: - events verbs: @@ -399,13 +411,6 @@ data: - create - update - patch - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - apiGroups: - snapshot.storage.k8s.io resources: @@ -425,6 +430,7 @@ data: - watch - update - delete + - patch - apiGroups: - snapshot.storage.k8s.io resources: @@ -449,6 +455,21 @@ data: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver @@ -511,14 +532,34 @@ data: app: ebs-csi-controller app.kubernetes.io/name: aws-ebs-csi-driver spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 containers: - args: - - --endpoint=$(CSI_ENDPOINT) - - --logtostderr - - --v=2 + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' env: + - name: AWS_REGION + value: '${AWS_REGION}' - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' - name: CSI_NODE_NAME valueFrom: fieldRef: @@ -535,7 +576,14 @@ data: key: access_key name: aws-secret optional: true - image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.2.0 + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 5 @@ -558,69 +606,144 @@ data: initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir - args: - - --csi-address=$(ADDRESS) - - --v=2 - - --feature-gates=Topology=true - - --extra-create-metadata - - --leader-election=true - - --default-fstype=ext4 + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: registry.k8s.io/sig-storage/csi-provisioner:v2.1.1 + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir - args: - - --csi-address=$(ADDRESS) - - --v=2 - - --leader-election=true + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: registry.k8s.io/sig-storage/csi-attacher:v3.1.0 + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir - args: - - --csi-address=$(ADDRESS) - - --leader-election=true + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: registry.k8s.io/sig-storage/csi-snapshotter:v3.0.3 + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir - args: - - --csi-address=$(ADDRESS) - - --v=2 + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: registry.k8s.io/sig-storage/csi-resizer:v1.0.0 - imagePullPolicy: Always + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir - args: - - --csi-address=/csi/csi.sock - image: registry.k8s.io/sig-storage/livenessprobe:v2.2.0 + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true volumeMounts: - mountPath: /csi name: socket-dir nodeSelector: kubernetes.io/os: linux priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 serviceAccountName: ebs-csi-controller-sa tolerations: - key: CriticalAddonsOnly @@ -632,21 +755,11 @@ data: effect: NoSchedule - effect: NoSchedule key: node-role.kubernetes.io/control-plane - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists volumes: - emptyDir: {} name: socket-dir --- - apiVersion: policy/v1beta1 + apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -690,17 +803,19 @@ data: containers: - args: - node - - --endpoint=$(CSI_ENDPOINT) - - --logtostderr - - --v=2 + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' env: - name: CSI_ENDPOINT - value: unix:/csi/csi.sock + value: 'unix:/csi/csi.sock' - name: CSI_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.2.0 + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 5 httpGet: @@ -714,8 +829,16 @@ data: - containerPort: 9808 name: healthz protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi securityContext: privileged: true + readOnlyRootFilesystem: true volumeMounts: - mountPath: /var/lib/kubelet mountPropagation: Bidirectional @@ -725,38 +848,63 @@ data: - mountPath: /dev name: device-dir - args: - - --csi-address=$(ADDRESS) - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --v=2 + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' env: - name: ADDRESS value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.1.0 + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true volumeMounts: - mountPath: /csi name: plugin-dir - mountPath: /registration name: registration-dir - args: - - --csi-address=/csi/csi.sock - image: registry.k8s.io/sig-storage/livenessprobe:v2.2.0 + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true volumeMounts: - mountPath: /csi name: plugin-dir nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 serviceAccountName: ebs-csi-node-sa tolerations: - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - tolerationSeconds: 300 + - operator: Exists volumes: - hostPath: path: /var/lib/kubelet @@ -787,6 +935,7 @@ data: name: ebs.csi.aws.com spec: attachRequired: true + fsGroupPolicy: File podInfoOnMount: false kind: ConfigMap metadata: diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-gpu.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-gpu.yaml index d040658b71..c36a1177fd 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-gpu.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-gpu.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external gpu: nvidia name: ${CLUSTER_NAME} spec: @@ -36,19 +38,19 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-provider: aws + cloud-provider: external controllerManager: extraArgs: - cloud-provider: aws + cloud-provider: external initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' machineTemplate: infrastructureRef: @@ -72,25 +74,10 @@ spec: type: gp2 sshKeyName: ${AWS_SSH_KEY_NAME} --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: crs-gpu-operator -spec: - clusterSelector: - matchLabels: - gpu: nvidia - resources: - - kind: ConfigMap - name: nvidia-clusterpolicy-crd - - kind: ConfigMap - name: nvidia-gpu-operator-components - strategy: ApplyOnce ---- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: - name: ${CLUSTER_NAME}-md + name: ${CLUSTER_NAME}-md-0 spec: clusterName: ${CLUSTER_NAME} replicas: ${WORKER_MACHINE_COUNT} @@ -102,18 +89,18 @@ spec: configRef: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md + name: ${CLUSTER_NAME}-md-0 clusterName: ${CLUSTER_NAME} infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachineTemplate - name: ${CLUSTER_NAME}-md + name: ${CLUSTER_NAME}-md-0 version: ${KUBERNETES_VERSION} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachineTemplate metadata: - name: ${CLUSTER_NAME}-md + name: ${CLUSTER_NAME}-md-0 spec: template: spec: @@ -127,14 +114,14 @@ spec: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: - name: ${CLUSTER_NAME}-md + name: ${CLUSTER_NAME}-md-0 spec: template: spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' --- apiVersion: v1 @@ -156,6 +143,1031 @@ spec: name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-gpu-operator +spec: + clusterSelector: + matchLabels: + gpu: nvidia + resources: + - kind: ConfigMap + name: nvidia-clusterpolicy-crd + - kind: ConfigMap + name: nvidia-gpu-operator-components + strategy: ApplyOnce +--- apiVersion: v1 data: clusterpolicy-crd.yaml: | @@ -4353,6 +5365,9 @@ data: - name: host-sys mountPath: "/host-sys" readOnly: true + - name: host-usr-lib + mountPath: "/host-usr/lib" + readOnly: true - name: source-d mountPath: "/etc/kubernetes/node-feature-discovery/source.d/" readOnly: true @@ -4372,6 +5387,9 @@ data: - name: host-sys hostPath: path: "/sys" + - name: host-usr-lib + hostPath: + path: "/usr/lib" - name: source-d hostPath: path: "/etc/kubernetes/node-feature-discovery/source.d/" @@ -4440,7 +5458,7 @@ data: enabled: true repository: nvcr.io/nvidia image: driver - version: 525.60.13-ubuntu20.04 + version: 525.60.13 imagePullPolicy: IfNotPresent rdma: enabled: false @@ -4801,12 +5819,6 @@ data: - get - list - watch - {{- if or (.Values.operator.cleanupCRD) (.Values.operator.upgradeCRD) }} - - delete - - create - - update - - patch - {{- end }} --- # Source: gpu-operator/templates/rolebinding.yaml kind: ClusterRoleBinding @@ -4815,6 +5827,7 @@ data: name: gpu-operator labels: app.kubernetes.io/component: "gpu-operator" + subjects: - kind: ServiceAccount name: gpu-operator diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-ignition.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-ignition.yaml index e3b993858c..25330e2968 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-ignition.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-ignition.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external name: ${CLUSTER_NAME} spec: clusterNetwork: @@ -40,10 +42,10 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-provider: aws + cloud-provider: external controllerManager: extraArgs: - cloud-provider: aws + cloud-provider: external format: ignition ignition: containerLinuxConfig: @@ -65,12 +67,12 @@ spec: initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: $${COREOS_EC2_HOSTNAME} joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: $${COREOS_EC2_HOSTNAME} preKubeadmCommands: - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp @@ -158,7 +160,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: $${COREOS_EC2_HOSTNAME} preKubeadmCommands: - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp @@ -182,3 +184,1013 @@ spec: - kind: ConfigMap name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-internal-elb.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-internal-elb.yaml index fc85dc790e..9d5245abd3 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-internal-elb.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-internal-elb.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external name: ${CLUSTER_NAME} spec: clusterNetwork: @@ -43,19 +45,19 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-provider: aws + cloud-provider: external controllerManager: extraArgs: - cloud-provider: aws + cloud-provider: external initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' preKubeadmCommands: - mkdir -p /opt/cluster-api @@ -76,6 +78,7 @@ metadata: spec: template: spec: + failureDomain: us-west-2a iamInstanceProfile: control-plane.cluster-api-provider-aws.sigs.k8s.io instanceType: ${AWS_CONTROL_PLANE_MACHINE_TYPE} sshKeyName: ${AWS_SSH_KEY_NAME} @@ -97,7 +100,6 @@ spec: kind: KubeadmConfigTemplate name: ${CLUSTER_NAME}-md-0 clusterName: ${CLUSTER_NAME} - failureDomain: us-west-2a infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachineTemplate @@ -111,6 +113,7 @@ metadata: spec: template: spec: + failureDomain: us-west-2a iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io instanceType: ${AWS_NODE_MACHINE_TYPE} sshKeyName: ${AWS_SSH_KEY_NAME} @@ -125,7 +128,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' preKubeadmCommands: - ctr -n k8s.io images pull "${CAPI_IMAGES_REGISTRY}:${E2E_IMAGE_TAG}" @@ -149,3 +152,1013 @@ spec: - kind: ConfigMap name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-intree-cloud-provider.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-intree-cloud-provider.yaml new file mode 100644 index 0000000000..4af34735f3 --- /dev/null +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-intree-cloud-provider.yaml @@ -0,0 +1,135 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cni: ${CLUSTER_NAME}-crs-0 + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSCluster +metadata: + name: ${CLUSTER_NAME} +spec: + region: ${AWS_REGION} + sshKeyName: ${AWS_SSH_KEY_NAME} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: aws + controllerManager: + extraArgs: + cloud-provider: aws + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: aws + name: '{{ ds.meta_data.local_hostname }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: aws + name: '{{ ds.meta_data.local_hostname }}' + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + iamInstanceProfile: control-plane.cluster-api-provider-aws.sigs.k8s.io + instanceType: ${AWS_CONTROL_PLANE_MACHINE_TYPE} + sshKeyName: ${AWS_SSH_KEY_NAME} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io + instanceType: ${AWS_NODE_MACHINE_TYPE} + sshKeyName: ${AWS_SSH_KEY_NAME} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: aws + name: '{{ ds.meta_data.local_hostname }}' +--- +apiVersion: v1 +data: ${CNI_RESOURCES} +kind: ConfigMap +metadata: + name: cni-${CLUSTER_NAME}-crs-0 +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-crs-0 +spec: + clusterSelector: + matchLabels: + cni: ${CLUSTER_NAME}-crs-0 + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-crs-0 + strategy: ApplyOnce diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-kcp-remediation.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-kcp-remediation.yaml index cc7918eca5..f741189cc0 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-kcp-remediation.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-kcp-remediation.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external name: ${CLUSTER_NAME} spec: clusterNetwork: @@ -38,19 +40,19 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-provider: aws + cloud-provider: external controllerManager: extraArgs: - cloud-provider: aws + cloud-provider: external initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' machineTemplate: infrastructureRef: @@ -115,7 +117,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' --- apiVersion: v1 @@ -137,6 +139,1016 @@ spec: name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: @@ -144,9 +1156,11 @@ metadata: spec: clusterName: ${CLUSTER_NAME} maxUnhealthy: 100% + nodeStartupTimeout: 30s selector: matchLabels: cluster.x-k8s.io/control-plane: "" + mhc-test: fail unhealthyConditions: - status: "False" timeout: 10s diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-kcp-scale-in.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-kcp-scale-in.yaml index ca9f8dc1f0..61ce51d77b 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-kcp-scale-in.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-kcp-scale-in.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external name: ${CLUSTER_NAME} spec: clusterNetwork: @@ -38,19 +40,19 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-provider: aws + cloud-provider: external controllerManager: extraArgs: - cloud-provider: aws + cloud-provider: external initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' machineTemplate: infrastructureRef: @@ -118,7 +120,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' --- apiVersion: v1 @@ -139,3 +141,1013 @@ spec: - kind: ConfigMap name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-limit-az.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-limit-az.yaml index d7c2ab00e6..e459af0305 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-limit-az.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-limit-az.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external name: ${CLUSTER_NAME} spec: clusterNetwork: @@ -38,19 +40,19 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-provider: aws + cloud-provider: external controllerManager: extraArgs: - cloud-provider: aws + cloud-provider: external initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' machineTemplate: infrastructureRef: @@ -115,7 +117,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' --- apiVersion: v1 @@ -136,3 +138,1013 @@ spec: - kind: ConfigMap name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-machine-pool.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-machine-pool.yaml index c9fe833d6c..e916ebb5d7 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-machine-pool.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-machine-pool.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external name: ${CLUSTER_NAME} spec: clusterNetwork: @@ -38,19 +40,19 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-provider: aws + cloud-provider: external controllerManager: extraArgs: - cloud-provider: aws + cloud-provider: external initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' machineTemplate: infrastructureRef: @@ -112,7 +114,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' --- apiVersion: cluster.x-k8s.io/v1beta1 @@ -131,12 +133,12 @@ spec: name: ${CLUSTER_NAME}-mp-1 clusterName: ${CLUSTER_NAME} infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachinePool name: ${CLUSTER_NAME}-mp-1 version: ${KUBERNETES_VERSION} --- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachinePool metadata: name: ${CLUSTER_NAME}-mp-1 @@ -148,7 +150,7 @@ spec: maxPrice: "" sshKeyName: ${AWS_SSH_KEY_NAME} maxSize: 4 - minSize: 1 + minSize: 0 --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfig @@ -158,7 +160,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' --- apiVersion: v1 @@ -179,3 +181,1013 @@ spec: - kind: ConfigMap name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-md-remediation.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-md-remediation.yaml index 0dab6b8931..9b67d9af7c 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-md-remediation.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-md-remediation.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external name: ${CLUSTER_NAME} spec: clusterNetwork: @@ -38,19 +40,19 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-provider: aws + cloud-provider: external controllerManager: extraArgs: - cloud-provider: aws + cloud-provider: external initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' machineTemplate: infrastructureRef: @@ -117,7 +119,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' --- apiVersion: v1 @@ -139,6 +141,1016 @@ spec: name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-multi-az.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-multi-az.yaml index 368fabcfb1..a6e2b18676 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-multi-az.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-multi-az.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external name: ${CLUSTER_NAME} spec: clusterNetwork: @@ -47,19 +49,19 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-provider: aws + cloud-provider: external controllerManager: extraArgs: - cloud-provider: aws + cloud-provider: external initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' machineTemplate: infrastructureRef: @@ -124,7 +126,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' --- apiVersion: v1 @@ -145,3 +147,1013 @@ spec: - kind: ConfigMap name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-nested-multitenancy.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-nested-multitenancy.yaml index cd54f40466..cab93da437 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-nested-multitenancy.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-nested-multitenancy.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external name: ${CLUSTER_NAME} spec: clusterNetwork: @@ -43,19 +45,19 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-provider: aws + cloud-provider: external controllerManager: extraArgs: - cloud-provider: aws + cloud-provider: external initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' machineTemplate: infrastructureRef: @@ -128,7 +130,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' --- apiVersion: v1 @@ -150,6 +152,1016 @@ spec: name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon +--- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSClusterRoleIdentity metadata: diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-peered-remote.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-peered-remote.yaml index 72e24669be..e4304c6196 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-peered-remote.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-peered-remote.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external name: ${CLUSTER_NAME} spec: clusterNetwork: @@ -48,19 +50,19 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-provider: aws + cloud-provider: external controllerManager: extraArgs: - cloud-provider: aws + cloud-provider: external initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' preKubeadmCommands: - mkdir -p /opt/cluster-api @@ -111,7 +113,6 @@ spec: kind: KubeadmConfigTemplate name: ${CLUSTER_NAME}-md-0 clusterName: ${CLUSTER_NAME} - failureDomain: us-west-2a infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachineTemplate @@ -125,6 +126,7 @@ metadata: spec: template: spec: + failureDomain: us-west-2a iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io instanceType: ${AWS_NODE_MACHINE_TYPE} sshKeyName: ${AWS_SSH_KEY_NAME} @@ -139,7 +141,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' preKubeadmCommands: - ctr -n k8s.io images pull "${CAPI_IMAGES_REGISTRY}:${E2E_IMAGE_TAG}" @@ -163,3 +165,1013 @@ spec: - kind: ConfigMap name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-remote-management-cluster.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-remote-management-cluster.yaml index b891668181..d75a7c4768 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-remote-management-cluster.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-remote-management-cluster.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external name: ${CLUSTER_NAME} spec: clusterNetwork: @@ -38,19 +40,19 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-provider: aws + cloud-provider: external controllerManager: extraArgs: - cloud-provider: aws + cloud-provider: external initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' preKubeadmCommands: - mkdir -p /opt/cluster-api @@ -119,7 +121,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' preKubeadmCommands: - ctr -n k8s.io images pull "${CAPI_IMAGES_REGISTRY}:${E2E_IMAGE_TAG}" @@ -143,3 +145,1013 @@ spec: - kind: ConfigMap name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-simple-multitenancy.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-simple-multitenancy.yaml index 53a3492671..2668643ac5 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-simple-multitenancy.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-simple-multitenancy.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external name: ${CLUSTER_NAME} spec: clusterNetwork: @@ -41,19 +43,19 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-provider: aws + cloud-provider: external controllerManager: extraArgs: - cloud-provider: aws + cloud-provider: external initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' machineTemplate: infrastructureRef: @@ -118,7 +120,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' --- apiVersion: v1 @@ -140,6 +142,1016 @@ spec: name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon +--- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSClusterRoleIdentity metadata: diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-spot-instances.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-spot-instances.yaml index 971730d240..e7fd3ff162 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-spot-instances.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-spot-instances.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external name: ${CLUSTER_NAME} spec: clusterNetwork: @@ -35,19 +37,19 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-provider: aws + cloud-provider: external controllerManager: extraArgs: - cloud-provider: aws + cloud-provider: external initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' machineTemplate: infrastructureRef: @@ -114,7 +116,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' --- apiVersion: v1 @@ -135,3 +137,1013 @@ spec: - kind: ConfigMap name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-ssm.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-ssm.yaml index 5f8650e311..ef4e6c5921 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-ssm.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-ssm.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external name: ${CLUSTER_NAME} spec: clusterNetwork: @@ -37,19 +39,19 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-provider: aws + cloud-provider: external controllerManager: extraArgs: - cloud-provider: aws + cloud-provider: external initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' machineTemplate: infrastructureRef: @@ -118,7 +120,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' --- apiVersion: v1 @@ -139,3 +141,1013 @@ spec: - kind: ConfigMap name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-upgrade-to-external-cloud-provider.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-upgrade-to-external-cloud-provider.yaml new file mode 100644 index 0000000000..9b76c906f5 --- /dev/null +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-upgrade-to-external-cloud-provider.yaml @@ -0,0 +1,1148 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + ccm: external + cni: ${CLUSTER_NAME}-crs-0 + csi: external + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSCluster +metadata: + name: ${CLUSTER_NAME} +spec: + region: ${AWS_REGION} + sshKeyName: ${AWS_SSH_KEY_NAME} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + controllerManager: + extraArgs: + cloud-provider: external + external-cloud-volume-plugin: aws + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data.local_hostname }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data.local_hostname }}' + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + iamInstanceProfile: control-plane.cluster-api-provider-aws.sigs.k8s.io + instanceType: ${AWS_CONTROL_PLANE_MACHINE_TYPE} + sshKeyName: ${AWS_SSH_KEY_NAME} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io + instanceType: ${AWS_NODE_MACHINE_TYPE} + sshKeyName: ${AWS_SSH_KEY_NAME} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data.local_hostname }}' +--- +apiVersion: v1 +data: ${CNI_RESOURCES} +kind: ConfigMap +metadata: + name: cni-${CLUSTER_NAME}-crs-0 +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-crs-0 +spec: + clusterSelector: + matchLabels: + cni: ${CLUSTER_NAME}-crs-0 + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-crs-0 + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-upgrade-to-main.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-upgrade-to-main.yaml index 1769569296..f451cf2f0d 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-upgrade-to-main.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-upgrade-to-main.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external name: ${CLUSTER_NAME} spec: clusterNetwork: @@ -35,19 +37,19 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-provider: aws + cloud-provider: external controllerManager: extraArgs: - cloud-provider: aws + cloud-provider: external initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' machineTemplate: infrastructureRef: @@ -112,7 +114,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' --- apiVersion: v1 @@ -134,6 +136,1016 @@ spec: name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon +--- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachineTemplate metadata: diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-upgrades.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-upgrades.yaml index d21b800a1a..87ddf0197e 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-upgrades.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template-upgrades.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external name: ${CLUSTER_NAME} spec: clusterNetwork: @@ -35,19 +37,19 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-provider: aws + cloud-provider: external controllerManager: extraArgs: - cloud-provider: aws + cloud-provider: external initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' machineTemplate: infrastructureRef: @@ -112,7 +114,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' --- apiVersion: v1 @@ -134,6 +136,1016 @@ spec: name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce --- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachinePool metadata: @@ -175,5 +1187,5 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' diff --git a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template.yaml b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template.yaml index 4af34735f3..79e65ce372 100644 --- a/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template.yaml +++ b/test/e2e/data/infrastructure-aws/withoutclusterclass/e2e_test_templates/cluster-template.yaml @@ -2,7 +2,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + ccm: external cni: ${CLUSTER_NAME}-crs-0 + csi: external name: ${CLUSTER_NAME} spec: clusterNetwork: @@ -35,19 +37,19 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-provider: aws + cloud-provider: external controllerManager: extraArgs: - cloud-provider: aws + cloud-provider: external initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' machineTemplate: infrastructureRef: @@ -112,7 +114,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' --- apiVersion: v1 @@ -133,3 +135,1013 @@ spec: - kind: ConfigMap name: cni-${CLUSTER_NAME}-crs-0 strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-node-critical + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 + args: + - --v=2 + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role + rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + containers: + - args: + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: AWS_REGION + value: '${AWS_REGION}' + - name: CSI_ENDPOINT + value: 'unix:///var/lib/csi/sockets/pluginproxy/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--feature-gates=Topology=true' + - '--extra-create-metadata' + - '--leader-election=true' + - '--default-fstype=ext4' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-provisioner:v3.4.0' + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-attacher:v4.2.0' + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--leader-election=true' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1' + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--v=2' + - '--handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-resizer:v1.7.0' + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - '--endpoint=$(CSI_ENDPOINT)' + - '--logging-format=text' + - '--v=2' + env: + - name: CSI_ENDPOINT + value: 'unix:/csi/csi.sock' + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: null + image: 'registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.17.0' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - '--csi-address=$(ADDRESS)' + - '--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)' + - '--v=2' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + envFrom: null + image: 'registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0' + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - '--csi-address=/csi/csi.sock' + envFrom: null + image: 'registry.k8s.io/sig-storage/livenessprobe:v2.9.0' + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon