Scalable jenkins

.gitignore

@@ -0,0 +1,8 @@
+.terraform
+*.tfstate
+planfile
+aks-engine-template.json
+_output
+translations
+.env
+.DS_Store

kubernetes/acc-k8s-cluster.json

@@ -0,0 +1,87 @@
+{
+  "apiVersion": "vlabs",
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "Kubernetes",
+      "kubernetesConfig": {
+        "addons": [
+          {
+            "name": "tiller",
+            "enabled" : true
+          },
+          {
+            "name": "cluster-autoscaler",
+            "enabled": true,
+            "pools": [
+              {
+                "name": "agentsbionic",
+                "config": {
+                  "min-nodes": "1",
+                  "max-nodes": "5"
+                }
+              },
+              {
+                "name": "agentsxenial",
+                "config": {
+                  "min-nodes": "1",
+                  "max-nodes": "5"
+                }
+              }
+            ],
+            "config": {
+              "scan-interval": "1m"
+            }
+          }
+        ]
+      }
+    },
+    "masterProfile": {
+      "count": 1,
+      "vmSize": "Standard_D2s_v3",
+      "vnetSubnetId": "${VNET_SUBNET_ID}",
+      "dnsPrefix": "${DNS_PREFIX}",
+      "firstConsecutiveStaticIP": "${FIRST_CONSECUTIVE_IP}"
+    },
+    "agentPoolProfiles": [
+      {
+        "name": "agentsbionic",
+        "count": 2,
+        "availabilityProfile": "VirtualMachineScaleSets",
+        "distro": "aks-ubuntu-18.04",
+        "vmSize": "${ACC_VM_SIZE}",
+        "vnetSubnetId": "${VNET_SUBNET_ID}",
+        "storageProfile": "ManagedDisks"
+      },
+      {
+        "name": "agentsxenial",
+        "count": 2,
+        "availabilityProfile": "VirtualMachineScaleSets",
+        "distro": "acc-16.04",
+        "vmSize": "${ACC_VM_SIZE}",
+        "vnetSubnetId": "${VNET_SUBNET_ID}",
+        "storageProfile": "ManagedDisks"
+      }
+    ],
+    "linuxProfile": {
+      "adminUsername": "azureuser",
+      "ssh": {
+        "publicKeys": [
+          {
+            "keyData": "${SSH_PUBLIC_KEY}"
+          }
+        ]
+      }
+    },
+    "extensionProfiles": [
+      {
+        "name": "prometheus-grafana-k8s",
+        "version": "v1"
+      }
+    ],
+    "servicePrincipalProfile": {
+      "clientId": "${SERVICE_PRINCIPAL_ID}",
+      "secret": "${SERVICE_PRINCIPAL_PASSWORD}"
+
+    }
+  }
+}

kubernetes/admin-user.yml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: admin-user
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: admin-user
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: admin-user
+  namespace: kube-system

kubernetes/deploy-k8s.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+# Copyright (c) Open Enclave SDK contributors.
+# Licensed under the MIT License.
+
+set -o errexit
+
+if [[ -z $SUBSCRIPTION_ID ]]; then echo "ERROR: Env variable SUBSCRIPTION_ID is not set"; exit 1; fi
+if [[ -z $SERVICE_PRINCIPAL_ID ]]; then echo "ERROR: Env variable SERVICE_PRINCIPAL_ID is not set"; exit 1; fi
+if [[ -z $SERVICE_PRINCIPAL_PASSWORD ]]; then echo "ERROR: Env variable SERVICE_PRINCIPAL_PASSWORD is not set"; exit 1; fi
+if [[ -z $TENANT_ID ]]; then echo "ERROR: Env variable TENANT_ID is not set"; exit 1; fi
+if [[ -z $REGION ]]; then echo "ERROR: Env variable REGION is not set"; exit 1; fi
+if [[ -z $RESOURCE_GROUP ]]; then echo "ERROR: Env variable RESOURCE_GROUP is not set"; exit 1; fi
+if [[ -z $VNET_SUBNET_ID ]]; then echo "ERROR: Env variable VNET_SUBNET_ID is not set"; exit 1; fi
+if [[ -z $DNS_PREFIX ]]; then echo "ERROR: Env variable DNS_PREFIX is not set"; exit 1; fi
+if [[ -z $ACC_VM_SIZE ]]; then echo "ERROR: Env variable ACC_VM_SIZE is not set"; exit 1; fi
+if [[ -z $KV_NAME ]]; then echo "ERROR: Env variable KV_NAME is not set"; exit 1; fi
+if [[ -z $KV_SECRET_SSH_PUB ]]; then echo "ERROR: Env variable KV_SECRET_SSH_PUB is not set"; exit 1; fi
+if [[ -z $KV_SECRET_WIN_PWD ]]; then echo "ERROR: Env variable KV_SECRET_WIN_PWD is not set"; exit 1; fi
+
+
+#
+# Create the Azure ACC Kubernetes cluster via aks-engine
+#
+az login --service-principal -u "${SERVICE_PRINCIPAL_ID}" -p "${SERVICE_PRINCIPAL_PASSWORD}" --tenant "${TENANT_ID}" --output table
+az account set --subscription "${SUBSCRIPTION_ID}"
+
+KEY=$(az keyvault secret show --vault-name "${KV_NAME}" --name "${KV_SECRET_SSH_PUB}" | jq -r .value | base64 -d)
+PASSWORD=$(az keyvault secret show --vault-name "${KV_NAME}" --name "${KV_SECRET_WIN_PWD}" | jq -r .value)
+
+export WINDOWS_ADMIN_PASSWORD="$PASSWORD"
+export SSH_PUBLIC_KEY="$KEY"
+
+TEMPLATE="acc-k8s-cluster.json"
+
+
+DIR=$(dirname "$0")
+cd "$DIR"
+eval "cat << EOF
+$(cat "$TEMPLATE")
+EOF
+" > aks-engine-template.json
+aks-engine generate aks-engine-template.json
+RG_EXISTS=$(az group exists --name "$RESOURCE_GROUP")
+if [[ "$RG_EXISTS" = "false" ]]; then
+   az group create --name "$RESOURCE_GROUP" --location "$REGION" --output table
+fi
+az group deployment create --name acc-k8s \
+                          --resource-group ${RESOURCE_GROUP} \
+                          --template-file _output/${DNS_PREFIX}/azuredeploy.json\
+                          --parameters @_output/${DNS_PREFIX}/azuredeploy.parameters.json \
+                          --output table
+
+export KUBECONFIG=_output/${DNS_PREFIX}/kubeconfig/kubeconfig.${REGION}.json
+kubectl get nodes
+kubectl apply -f "admin-user.yml"
+
+az keyvault secret set --vault-name "${KV_NAME}" --name "kubeconfig-${DNS_PREFIX}-${REGION}" --file ${KUBECONFIG} --description "${DNS_PREFIX}.${REGION}.cloudapp.azure.com Kubeconfig"
+
+echo "KUBECONFIG file successfully uploaded to oe-ci-test-kv keyvault"
+exit 0

kubernetes/jenkins.yml

@@ -0,0 +1,124 @@
+---
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  name: jenkins
+  labels:
+    name: jenkins
+spec:
+  serviceName: jenkins
+  replicas: 1
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      name: jenkins
+      labels:
+        name: jenkins
+    spec:
+      terminationGracePeriodSeconds: 10
+      serviceAccountName: jenkins
+      containers:
+        - name: jenkins
+          image: jenkins/jenkins:lts-alpine
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 8080
+            - containerPort: 50000
+          resources:
+            limits:
+              cpu: 1
+              memory: 1Gi
+            requests:
+              cpu: 0.5
+              memory: 500Mi
+          env:
+            - name: LIMITS_MEMORY
+              valueFrom:
+                resourceFieldRef:
+                  resource: limits.memory
+                  divisor: 1Mi
+            - name: JAVA_OPTS
+              # value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=1 -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
+              value: -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
+          volumeMounts:
+            - name: jenkins-home
+              mountPath: /var/jenkins_home
+          livenessProbe:
+            httpGet:
+              path: /login
+              port: 8080
+            initialDelaySeconds: 60
+            timeoutSeconds: 5
+            failureThreshold: 12 # ~2 minutes
+          readinessProbe:
+            httpGet:
+              path: /login
+              port: 8080
+            initialDelaySeconds: 60
+            timeoutSeconds: 5
+            failureThreshold: 12 # ~2 minutes
+      securityContext:
+        fsGroup: 1000
+  volumeClaimTemplates:
+  - metadata:
+      name: jenkins-home
+      # annotations:
+      #   volume.beta.kubernetes.io/storage-class: anything
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 1Gi
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: jenkins
+spec:
+  # type: LoadBalancer
+  selector:
+    name: jenkins
+  # ensure the client ip is propagated to avoid the invalid crumb issue when using LoadBalancer (k8s >=1.7)
+  #externalTrafficPolicy: Local
+  ports:
+    -
+      name: http
+      port: 80
+      targetPort: 8080
+      protocol: TCP
+    -
+      name: agent
+      port: 50000
+      protocol: TCP
+
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: jenkins
+  annotations:
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    kubernetes.io/tls-acme: "true"
+    # "413 Request Entity Too Large" uploading plugins, increase client_max_body_size
+    nginx.ingress.kubernetes.io/proxy-body-size: 50m
+    nginx.ingress.kubernetes.io/proxy-request-buffering: "off"
+    # For nginx-ingress controller < 0.9.0.beta-18
+    ingress.kubernetes.io/ssl-redirect: "true"
+    # "413 Request Entity Too Large" uploading plugins, increase client_max_body_size
+    ingress.kubernetes.io/proxy-body-size: 50m
+    ingress.kubernetes.io/proxy-request-buffering: "off"
+spec:
+  rules:
+  - http:
+      paths:
+      - path: /
+        backend:
+          serviceName: jenkins
+          servicePort: 80
+    host: oe-jenkins.westeurope.cloudapp.azure.com
+  tls:
+  - hosts:
+    - oe-jenkins.westeurope.cloudapp.azure.com
+    secretName: tls-jenkins

kubernetes/service-account.yml

@@ -0,0 +1,37 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: jenkins
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: jenkins
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["create","delete","get","list","patch","update","watch"]
+- apiGroups: [""]
+  resources: ["pods/exec"]
+  verbs: ["create","delete","get","list","patch","update","watch"]
+- apiGroups: [""]
+  resources: ["pods/log"]
+  verbs: ["get","list","watch"]
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: jenkins
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: jenkins
+subjects:
+- kind: ServiceAccount
+  name: jenkins