Add self-deployment playbook
This adds a playbook and some Zuul secrets which will eventually
allow Zuul to deploy itself.
This also switches to using plain k8s objects instead of the
helm charts. The current system has too much local customization
for the helm charts, so just use plain k8s yaml.
Change-Id: I4e67be7976d421d211cfbfab014826d0cda299cf
diff --git a/.zuul.yaml b/.zuul.yaml
index 2a5dca9..9a5a9a3 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -2,3 +2,82 @@
check:
jobs:
- noop
+
+- secret:
+ name: zuul_deploy
+ data:
+ server: "https://10.0.0.1:443"
+ cert: "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURDekNDQWZPZ0F3SUJBZ0lRWW9OSEJiWlBvUXpEZ2JCMVNhOGFLekFOQmdrcWhraUc5dzBCQVFzRkFEQXYKTVMwd0t3WURWUVFERXlSbE5ETXhZVFU1WWkwNFptRTFMVFJsT1dRdFlqRTFZeTA1WmpNeE16UmpNR1psTUdFdwpIaGNOTVRreE1qSXdNVFl5TnpNMldoY05NalF4TWpFNE1UY3lOek0yV2pBdk1TMHdLd1lEVlFRREV5UmxORE14CllUVTVZaTA0Wm1FMUxUUmxPV1F0WWpFMVl5MDVaak14TXpSak1HWmxNR0V3Z2dFaU1BMEdDU3FHU0liM0RRRUIKQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUUNoS2k4cTJ5d2N4WWR4OHlQQmZzMXlGUmJuZ1ZXK29GUzFYV1FlKy9pTgo4a2M4K05IU3I2RGNFQUtGVExnMTU3VTNiOE5TdG9MdEdUNUtoV010MHh2NEcrQVc5Qy9GNzFzWWhscThGdXkxCkphQWZOeCtydlZEeW80YnJ2bU1ubXAwWmpMZzNIUjVBMThXT2xBZW52OGtyRVZ2VnNYL1hZbnMzb2FWeUhGYVEKNzdDcUNlYWpqeS9uLzdSUlBKNDhReVVvVTNRZDRmV0xXQ0c5emxhNzZ4eHlFSVh0MWlMVGZ5NElGYk9pbFlYcwpUVzMvZ1ZObTRjZHZaQmtjeDlPWkNNZXN5aHBEK1ZTUkpuVkdUdHA5UmZVNWlHc3JiejdTVzh6S3ovVjI4amV4CjEzY2V0cWw4Mk5PUGs4cTRleDJsZmJibGdqUTJBSWIrZWNtYUJHaWQxWmtMQWdNQkFBR2pJekFoTUE0R0ExVWQKRHdFQi93UUVBd0lDQkRBUEJnTlZIUk1CQWY4RUJUQURBUUgvTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFBawpuMEdRRGFYYW1Da2twY0ZneUI1bWlHd0s2NktLNnRqNG1Jc1ZDMG9DdWJ2bFQ0UDBFS2pJeDlad0hHeDJTK3ZiCnM1M2s4bmx5cHBtb1VYRXVxcUVLREpONlNMS3JhTFhWVGo4c1EwaExxbWltYTc4UmhTSVJMU3EraWJTMVlMSVcKakRvR3lzYXgzVE1mamEyRlU1ZC9IZXJHNHI5cWQxaXFMTnRiWU4xYWovUmJ3SUFvOFdqWE42WldkRDJic0hNaQpFVDFwbEQrSzhEb3E2R3MzbEt6c1k1Q1MyS1ZlSmVnWFhVc0dFMExIYjYwY3FvVldVMUhHc1NYTXJuTldrZVM0Cldrc0hSa0dpQVZBU21DMDZaZUM1d0lvSGNIYzJRTjhZTjdKTE9CV0FRUHhBRHlLc053OC9Eci8xS1VLdkZvZHkKSlhCNlhBS2pnUnFCdW9ROUJlMzkKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo="
+ token: !encrypted/pkcs1-oaep
+ - YWeClYtmP1RICp9ZoPQRBv80N/q70r3KEcqN64aDA9mq9O0kL7xGpUcCOLtsFoEG9bTtL
+ W/A9mv2Y1E2zEOVTKB2whdqkEA7kUbXht4pCc9FSqbg7/dJ4SyNy0b2tdCHfGc+9mLdRj
+ m2CLTYurUYQIDfQGlzjseFuv1RKX+8DDKVLas/Gsb6JGo6hBeq2CvryJHQnX8YPgz678g
+ UCQbFrUTpKmL0w+MJY8uP5I4WVBZtCzV1yGQHj+oxWLiZdVv7gJClm/y0gyndyZpWuGn1
+ Bx2dy0Y2ziKYBdV5Q9SXSHjA2PrZHU04B9Nkl2Hnez6Wqy/b1jQHMNbgmdzmmbQsbkjYS
+ e0wrjLxqXdZNNQJc9Fse7juPPVuaHtVdtZBSYPNyqjSPiWTUQDN09hULkq6XIT41TJF7g
+ Vh3Ti5+mDsD2VlqosXzOVsAilDyukIHTEVbJXpqsKY8EfPljRH6IPwUpvyWZsjrlktszY
+ qZedlLbBn/ke5Rkq9ULMakFQguniVaZ4VImU+VJOzbJlqXXzMO6OmEgwsgzEdOI4WMTKT
+ nu+YZdKtGQczh5j95+zpPC5PGyaZGEP6CQPPl1giOYoUHzoaYS1vagFhSoiY1pK3IarmF
+ 85kcLqvUtMcK/q591Ju02Hd7wiSJs0mE91BFeBTPZZBmMU9ePM0HV1oxj/Fy/8=
+ - UaJIBAJSdtakgx8AUecMKUx2ov5yX+E7lVn/xAz4Jqw5F6kgFBUJD3ht4zTPcwHxoVtR2
+ 32wzGon5X3Kqw9+mvSB5az9kKD5rbMB3Mgn01y4UGrMuGyFjPCaSUJDESSDqC0bXWqswY
+ 0+PhvzpJa5yE0ovmsR8wNJ8Ca8SfvcbRpRDe+h+r5LcBCG3cr3JkkbkaRkriH/ooWGrko
+ ESLQ42pGF7Z77v73zkFaQquKmV99logLwPnXnUYEkuxjaEAWLKc8aPAkEp//WVCuR+xKn
+ 2WClkdojrSu6GgPmIWd5/kS0lKXd+N5nEz3KaNXOYNvx9cIEVTWiLaWshbZX3KyTIahv8
+ qvqJrWD6Mw8Ug78QM4E0bwqxuk4l8SbQKoZjdOFQLqUu+ZEYOOSC45p0IcEHCxblP46E8
+ qEpzVBtliMjZ6MLi5M22IwRYJN5hr0L56MGCMg9b3vlDqPrX6Gz6p0FNKRZHekMSD5KtE
+ dwOFiwvzk9+GI2ZzxyjB+BSDLCr52OhgsK8lt+fUsc7/nDxnnop/tI76FcZ/5cm8DQGbS
+ wjzSbnwF0Qq5gdreQqcpGzF/NvKU2LDbHCO7sPxjI9+WOJQS4cy6J6EHmy5/jPAqaKG9K
+ 4yqqUj+70mc/0xd+sQP4UQWbO4kqWVjTg4b3o/Dof+3ghSTKEyD13q5KvbUzWo=
+ db_password: !encrypted/pkcs1-oaep
+ - kxXAI0xRUVstc8q/lloTpo5bs4HwecTJyip/Te2uYgSTDsFKJLipWEAkbb3ziA2/rdgbP
+ nJIiD1IYzjfqIE574LsJpqe5TssR5GMnsENRxxzjOJagSJJWUZJHBIU3EiLs1rrnr5TtY
+ 3wlFJvsdC1F5XNtuM17tFvgsH1ClsfrwjtcibQbYYm0dzBO1ZY7bqvl17AZPPE9G18G1O
+ FlfALHpkS1Cu84MY7Q8+jp33jftjaxxkPsOKLKoxtYkT4nn5MsDsvkstg+d7KiVl/bPKj
+ YKe5T2UzTBWak/P5Z7CQkfsuHe++sX4fHrdNgpKTp1EUfYYbMZ46vi9MM0sVN39RMlWHP
+ 72HqhHeuVbZH1uyi+P7xJKfmpZkbtiTCNQSmenuNypE2Dlwxu37VQlftCtYD4zP4xUSp5
+ 33z8+KRQmqJ3ases/VvbEdlxeuRG05CekYdenkGhqwcojefU8jcNQoqDBEfUAPH7Go+cB
+ aivnk+i+qvYcprAJ2YqXmLFLmI5//W1JD7k3te2M1KQuW/Wee7bb9RYeUGzeeG71g/reb
+ 0MP9FIAGK6HxBFWLiRS+Q0gktOEOlfPYdWGFQt+3r1kArARPj6M8vskMhrw+ncp9+zi7V
+ rNR1vRhoQieBzZtIYrgg7fqV/y7y28lLlRrxOMR7Vdk1WjJPDR57B8XV38oMHE=
+ nodepool_private_key: !encrypted/pkcs1-oaep
+ - qGQJDEhQizjE7WiZVEZlwuBwCkA6PbixWJxj6ijQmo7ARmo2yT+FHekwCPbuilUQoI4rV
+ Bzqn9uwbOJCveS1MeRag31gSsvwGwNsfuHNmjcpHacvJk22H3YZxSapjNAWK/csx17Avm
+ tvSVGZOO9U/di7z7uhnlHMHG1oDl6aaSwluQ+At6upsZNzdRNYNbngeBxsE9NwNFnNsqA
+ /V7jCtX1dCK9pSt1ptXUr/jrLnT7mI4ix/fS/bjYNlxc6T8Ouz6P3ZyE3Gwe38t8Oi87n
+ 6DE01YxH119Zv2T6xxZrRugkmr23fwkgKlZ8tp212EEiUo18MZv+FpjAQnbUNNEQkt7CO
+ SawL2cjg/TetPRtIhsmknkQVABc6ZP5edxZi87620kV4BoLbmmavFgKVMMtcAzntinGTp
+ zgo+Eidrtlt6JsyUATI/2MlZ+TIc7+0llV5QOspM4HzJgeyYj5ywjNKycUZfWiA8PN+8x
+ eIVQANsAQRT6GbOcbhw96mO6FeuaOaAsAwmSt2++xoaymx+TScYwDNFfr3ZBRG++34o2r
+ EFpIWEKgdDoLeDD/G8FrRZzlpzcy/QslvWbcQn9Qs/AdnhvMiEgSOk4ZZvkQi0Txu5/xr
+ jpHXFS3zzGiiwUVDiUvC115ZUR9TTQa0WQAmjiFcXWo1npLJvVjYlX6BNUEr7A=
+ - aL0DcCaiOlQ1nI1PtdsK6x/kxx6dCNWgAmLzIzqUFOmKNUckMBJvtzWg/qFa8smfyEMKE
+ yPLyXiA92f7geVK9kRasnuRqDPHWeeURLpkrV1PN65RBLobsW2dBcf3PTRb3/hBUsgplo
+ i4TCRHKF89xvHVcSKpLcqq0Y3A6R7jlJDou//SxjYJhw7qnSD+hI9v+5Fl+8jq9hUfwDN
+ 5/Y4ElgfskeuVrS6Kt88xJTqR4Epn5fjXej+VEu9k7e12m7Tt0Q7dTEWGh+uYnWVxbDUu
+ l2Bmc01BoUh0hFLvNSzDuORHWl4genRExaBfp+dNjRLHB1e7YYLuE1fDY+zU3lH7ik5jo
+ HcUEdueCj/X9oPAVj3F653gG9Z4hcc5a1itucfh+FcFnil4WA9TmjhIA7n3cCE7/bRLJG
+ xab63NFAqSBYCBTIWtQX+9DLGBn48LVHs7j6mhmpsco9Dt/JW0uL4eG4VhmwnawdVCehP
+ pAtcNBn2KrBoD7hM+18zdGJC57CorJ71McqN6rGT4ula+0ACGYZHRPuPdqhDQhjtrD2FV
+ fkRaqOVo7CS7hB9gQbbxFjB9tFQwMHAQiVYEmow1pRMrH84agxMMSUS377DK+BJvu88Ku
+ SBuDaxxq4NAG+eQORT2myh7wkDYoefAyxLJVAzy4EgH/tC7uOKE4+V3Mb5RXjc=
+ - E0/f19t+at0iV1duvEDi37kkQYmZjf1EmOQ7+42+J3laR9n8QY1Zzd9GqsXTyq2jVoA+F
+ Crvw8C1t6XZTudbxp7QQlYzOuSWCncZB2Tspe/W4r8STD8LEjncZcawbxPGIalIeoQ/BP
+ E4vEzgZXbaIPUwCHTQm6TzRhGViVlYhPrBWmSnWikcFUhMn1lUpwp1/5F+1aZV3H7HBds
+ dxp0oKad7M+uCtmzRvNufvr8667ZVaWzfOx8nisuZVwLM8Jau3kLdX4OyVbVFB18MK8j8
+ 0R2wDjG1ihsjwsekIuKkK1mkBUoR9zHQqwN1PTv6Ih43Wc1LXNSyin/TqYcwr+x/c374q
+ aiabasVcGgiOCqbtKBM8KJGC7CbSBrYnszDuiwBzmVbd1IcIj6vzxyO0Pypmy9Wu9o6UN
+ Stmwyv28brV90KC5+rJdiN63/u6uXvJN8lHjUUQ87PbQsSGGFlINprdNeU4bELhYImvjh
+ IT43YBl4JMF4fOQrlqSZwQzsAZ+xRT4pXf3V490LpIIDzUeYDvPZ/11vLoAr/77d2u2uN
+ M0VWknjEDoJa+CkeQ6df/OAwRMzdMIk45XmjFXsNC/eCRSv74GgD7Mpd2K1t8kBMOdmZ5
+ dljO1xVpo+z0J9z+dGVdIBXBFkO0+lamzNSBUXq64H3CPtShPJvIRlS8b9+wSc=
+ - USLP72RgSDDYXX748OPn0o1h/LyM5PLEigZQllhno/vx2gxhKwkaUlzrHHbxmvuvhAlIJ
+ 08IOfcySTzf1Gr1t+3iTBrpUPq/9qm5cuMMso+eI/9IMQ0W7vqQmiB8vWji57kuXsLIX5
+ SEQtBSC6IlJKLm8aJhjlWH0A2F3OqqpbH8vOpWnPMOnrPHGFB+KmqJUllkdCxDFtvcVVD
+ ChCcNORR+N6X3r7i/m6Niu3Ehro7ynOwUM7btYOXWMqOp1nUXbXdWim1/5sqcKvl76A++
+ lDDr/12mx+LCfLEEpMhpq6P4jFjhXCs9fs57DaE5CROmJO0iKKXHgyRShNC5o6h6Lb+rd
+ dGWUeiNXwEYQbCKPjJBJ+kzFtO3+zctzVwkI0TwYrF3pPa2fS/chB69JbPovYTf+Jx921
+ p4WYO9wvOPqrRD6VQetGDBaF3yZgqqAA6wPt2MM56+jrSq0O+eRRO6Xg99cY5wTFq9Tgs
+ pgkAfAdVnuvWAtXDnnmQacixdGCunSg+gNCyVW+UDMiNsB75I+QcEzAbnTIFzLMCZzOIK
+ UWg4OXFafQPI/+JvdVVgqDggURHi9arahXXEKb69xIi9BIYucNhKQ1NeAMjQpUAfm8krw
+ suzdXr2VOIJIcgVz3S9Kg1oiTgkY6PNdI/nuc3Aq1LzDeKykB0wGaulcpQ6H/o=
diff --git a/README b/README
index d1a7327..f18df48 100644
--- a/README
+++ b/README
@@ -39,19 +39,60 @@
kubectl create namespace cert-manager
kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v0.12.0/cert-manager.yaml
-kubectl apply -n cert-manager -f letsencrypt.yaml
+kubectl apply -n cert-manager -f letsencrypt.yaml
+
+# Install mariadb
+
+kubectl create namespace mariadb
+
+Use Google cloud click to deploy
+TODO: find a better HA sql database operator
+
+kubectl port-forward svc/mariadb-mariadb --namespace mariadb 3306
+mysql -h 127.0.0.1 -P 3306 -u root -p
+create database zuul;
+GRANT ALL PRIVILEGES ON zuul.* TO 'zuul'@'%' identified by '<password>' WITH GRANT OPTION;
# Install Zuul
gcloud compute addresses create zuul-static-ip --global
-
kubectl create namespace zuul
-kubectl create -n zuul secret generic nodepool-config --from-file=./nodepool/nodepool.yaml --from-file=./nodepool/clouds.yaml
+# Bind k8s service accounts to gcp service accounts
+kubectl create serviceaccount --namespace zuul logs
+kubectl create serviceaccount --namespace zuul nodepool
+kubectl create serviceaccount --namespace zuul zuul
-kubectl create -n zuul secret generic zuul-tenant-config --from-file=./zuul/main.yaml
+gcloud iam service-accounts add-iam-policy-binding \
+ --role roles/iam.workloadIdentityUser \
+ --member "serviceAccount:gerritcodereview-ci.svc.id.goog[zuul/logs]" \
+ zuul-logs@gerritcodereview-ci.iam.gserviceaccount.com
-kubectl apply -n argocd -f nodepool-app.yaml
-kubectl apply -n argocd -f zuul-app.yaml
+gcloud iam service-accounts add-iam-policy-binding \
+ --role roles/iam.workloadIdentityUser \
+ --member "serviceAccount:gerritcodereview-ci.svc.id.goog[zuul/nodepool]" \
+ nodepool@gerritcodereview-ci.iam.gserviceaccount.com
-kubectl apply -n zuul -f ingress.yaml
+gcloud iam service-accounts add-iam-policy-binding \
+ --role roles/iam.workloadIdentityUser \
+ --member "serviceAccount:gerritcodereview-ci.svc.id.goog[zuul/zuul]" \
+ zuul-63@gerritcodereview-ci.iam.gserviceaccount.com
+
+kubectl annotate serviceaccount \
+ --namespace zuul logs \
+ iam.gke.io/gcp-service-account=zuul-logs@gerritcodereview-ci.iam.gserviceaccount.com
+
+kubectl annotate serviceaccount \
+ --namespace zuul nodepool \
+ iam.gke.io/gcp-service-account=nodepool@gerritcodereview-ci.iam.gserviceaccount.com
+
+kubectl annotate serviceaccount \
+ --namespace zuul zuul \
+ iam.gke.io/gcp-service-account=zuul-63@gerritcodereview-ci.iam.gserviceaccount.com
+
+# create a service account for self-deployment
+
+kubectl -n zuul create serviceaccount zuul-deployment
+kubectl create clusterrolebinding zuul-deployment-cluster-admin-binding \
+ --clusterrole cluster-admin \
+ --user system:serviceaccount:zuul:zuul-deployment
diff --git a/ingress.yaml b/ingress.yaml
deleted file mode 100644
index f678b17..0000000
--- a/ingress.yaml
+++ /dev/null
@@ -1,22 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Ingress
-metadata:
- name: zuul-web
- namespace: zuul
- annotations:
- kubernetes.io/ingress.global-static-ip-name: "zuul-static-ip"
- cert-manager.io/cluster-issuer: letsencrypt-prod
- acme.cert-manager.io/http01-edit-in-place: "true"
-spec:
- rules:
- - host: gerrit-zuul.inaugust.com
- http:
- paths:
- - backend:
- serviceName: zuul-web
- servicePort: 9000
- path: /*
- tls:
- - secretName: zuul-web-certs
- hosts:
- - gerrit-zuul.inaugust.com
diff --git a/k8s/authdaemon.yaml b/k8s/authdaemon.yaml
new file mode 100644
index 0000000..256002a
--- /dev/null
+++ b/k8s/authdaemon.yaml
@@ -0,0 +1,39 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+ name: gcloud-authdaemon
+ namespace: zuul
+ labels:
+ app.kubernetes.io/name: gcloud-authdaemon
+spec:
+ selector:
+ matchLabels:
+ name: gcloud-authdaemon
+ template:
+ metadata:
+ labels:
+ name: gcloud-authdaemon
+ spec:
+ tolerations:
+ - key: node-role.kubernetes.io/master
+ effect: NoSchedule
+ containers:
+ - name: gcloud-authdaemon
+ image: docker.io/jeblair/authdaemon:latest
+ resources:
+ limits:
+ memory: 200Mi
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ volumeMounts:
+ - name: authdaemon
+ mountPath: /authdaemon
+ terminationGracePeriodSeconds: 2
+ volumes:
+ - name: authdaemon
+ hostPath:
+ path: /var/authdaemon/executor
+ type: DirectoryOrCreate
+ serviceAccountName: logs
diff --git a/letsencrypt.yaml b/k8s/letsencrypt.yaml
similarity index 100%
rename from letsencrypt.yaml
rename to k8s/letsencrypt.yaml
diff --git a/k8s/nodepool.yaml b/k8s/nodepool.yaml
new file mode 100644
index 0000000..f3b2eab
--- /dev/null
+++ b/k8s/nodepool.yaml
@@ -0,0 +1,41 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ namespace: zuul
+ name: nodepool-launcher-gcs
+ labels:
+ app.kubernetes.io/name: nodepool
+ app.kubernetes.io/instance: nodepool
+ app.kubernetes.io/part-of: nodepool
+ app.kubernetes.io/component: nodepool-launcher
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app.kubernetes.io/name: nodepool
+ app.kubernetes.io/instance: nodepool
+ app.kubernetes.io/part-of: nodepool
+ app.kubernetes.io/component: nodepool-launcher
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/name: nodepool
+ app.kubernetes.io/instance: nodepool
+ app.kubernetes.io/part-of: nodepool
+ app.kubernetes.io/component: nodepool-launcher
+ spec:
+ containers:
+ - name: launcher
+ image: zuul/nodepool-launcher:latest
+ env:
+ - name: OS_CLIENT_CONFIG_FILE
+ value: /etc/nodepool/clouds.yaml
+ volumeMounts:
+ - name: nodepool-config
+ mountPath: /etc/nodepool
+ volumes:
+ - name: nodepool-config
+ secret:
+ secretName: nodepool-gcs
+ serviceAccountName: nodepool
diff --git a/k8s/zuul.yaml b/k8s/zuul.yaml
new file mode 100644
index 0000000..ed4eda8
--- /dev/null
+++ b/k8s/zuul.yaml
@@ -0,0 +1,301 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+ namespace: zuul
+ name: zuul-executor
+ labels:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ app.kubernetes.io/component: zuul-executor
+spec:
+ type: ClusterIP
+ clusterIP: None
+ ports:
+ - name: logs
+ port: 7900
+ protocol: TCP
+ targetPort: logs
+ selector:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ app.kubernetes.io/component: zuul-executor
+---
+apiVersion: v1
+kind: Service
+metadata:
+ namespace: zuul
+ name: zuul-gearman
+ labels:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ app.kubernetes.io/component: zuul-scheduler
+spec:
+ type: ClusterIP
+ ports:
+ - name: gearman
+ port: 4730
+ protocol: TCP
+ targetPort: gearman
+ selector:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ app.kubernetes.io/component: zuul-scheduler
+---
+apiVersion: v1
+kind: Service
+metadata:
+ namespace: zuul
+ name: zuul-web
+ labels:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ app.kubernetes.io/component: zuul-web
+spec:
+ type: NodePort
+ ports:
+ - name: zuul-web
+ port: 9000
+ protocol: TCP
+ targetPort: zuul-web
+ selector:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ app.kubernetes.io/component: zuul-web
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ namespace: zuul
+ name: zuul-merger
+ labels:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ app.kubernetes.io/component: zuul-merger
+spec:
+ replicas: 0
+ selector:
+ matchLabels:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ app.kubernetes.io/component: zuul-merger
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ app.kubernetes.io/component: zuul-merger
+ spec:
+ containers:
+ - name: merger
+ image: zuul/zuul-merger:latest
+ args:
+ - /usr/local/bin/zuul-merger
+ - -d
+ volumeMounts:
+ - name: zuul-config
+ mountPath: /etc/zuul
+ volumes:
+ - name: zuul-config
+ secret:
+ secretName: zuul-config
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ namespace: zuul
+ name: zuul-web
+ labels:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ app.kubernetes.io/component: zuul-web
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ app.kubernetes.io/component: zuul-web
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ app.kubernetes.io/component: zuul-web
+ spec:
+ containers:
+ - name: web
+ image: jeblair/zuul-web:latest
+ args:
+ - /usr/local/bin/zuul-web
+ - -d
+ ports:
+ - name: zuul-web
+ containerPort: 9000
+ volumeMounts:
+ - name: zuul-config
+ mountPath: /etc/zuul
+ volumes:
+ - name: zuul-config
+ secret:
+ secretName: zuul-config
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ namespace: zuul
+ name: zuul-executor
+ labels:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ app.kubernetes.io/component: zuul-executor
+spec:
+ serviceName: zuul-executor
+ replicas: 1
+ selector:
+ matchLabels:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ app.kubernetes.io/component: zuul-executor
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ app.kubernetes.io/component: zuul-executor
+ spec:
+ securityContext:
+ runAsUser: 1000
+ runAsGroup: 1000
+ containers:
+ - name: executor
+ image: jeblair/zuul-executor:latest
+ args:
+ - /usr/local/bin/zuul-executor
+ - -d
+ ports:
+ - name: logs
+ containerPort: 7900
+ volumeMounts:
+ - name: zuul-config
+ mountPath: /etc/zuul
+ - name: authdaemon
+ mountPath: /authdaemon
+ - name: zuul-var
+ mountPath: /var/lib/zuul
+ - name: nodepool-private-key
+ mountPath: /var/lib/zuul/ssh
+ securityContext:
+ privileged: true
+ volumes:
+ - name: zuul-var
+ emptyDir: {}
+ - name: zuul-config
+ secret:
+ secretName: zuul-config
+ - name: authdaemon
+ hostPath:
+ path: /var/authdaemon/executor
+ type: DirectoryOrCreate
+ - name: nodepool-private-key
+ secret:
+ secretName: nodepool-private-key
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ namespace: zuul
+ name: zuul-scheduler
+ labels:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ app.kubernetes.io/component: zuul-scheduler
+spec:
+ replicas: 1
+ serviceName: zuul-scheduler
+ selector:
+ matchLabels:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ app.kubernetes.io/component: zuul-scheduler
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ app.kubernetes.io/component: zuul-scheduler
+ spec:
+ containers:
+ - name: scheduler
+ image: jeblair/zuul-scheduler:latest
+ args:
+ - /usr/local/bin/zuul-scheduler
+ - -d
+ ports:
+ - name: gearman
+ containerPort: 4730
+ volumeMounts:
+ - name: zuul-config
+ mountPath: /etc/zuul
+ - name: zuul-tenant-config
+ mountPath: /etc/zuul/tenant
+ - name: zuul-scheduler
+ mountPath: /var/lib/zuul
+ volumes:
+ - name: zuul-config
+ secret:
+ secretName: zuul-config
+ - name: zuul-tenant-config
+ secret:
+ secretName: zuul-tenant-config
+ serviceAccountName: zuul
+ volumeClaimTemplates:
+ - metadata:
+ name: zuul-scheduler
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 80Gi
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+ name: zuul-web
+ namespace: zuul
+ annotations:
+ kubernetes.io/ingress.global-static-ip-name: "zuul-static-ip"
+ cert-manager.io/cluster-issuer: letsencrypt-prod
+ acme.cert-manager.io/http01-edit-in-place: "true"
+spec:
+ rules:
+ - host: gerrit-zuul.inaugust.com
+ http:
+ paths:
+ - backend:
+ serviceName: zuul-web
+ servicePort: 9000
+ path: /*
+ tls:
+ - secretName: zuul-web-certs
+ hosts:
+ - gerrit-zuul.inaugust.com
diff --git a/nodepool-app.yaml b/nodepool-app.yaml
deleted file mode 100644
index dfe4413..0000000
--- a/nodepool-app.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
- name: nodepool
- finalizers:
- - resources-finalizer.argocd.argoproj.io
-spec:
- project: default
- source:
- repoURL: https://github.com/jeblair/zuul-helm
- path: charts/nodepool
- helm:
- values: |
- clouds: ''
- builder:
- enabled: false
- config:
- secret: nodepool-config
- destination:
- namespace: zuul
- server: https://kubernetes.default.svc
diff --git a/nodepool/clouds.yaml b/nodepool/clouds.yaml
deleted file mode 100644
index e69de29..0000000
--- a/nodepool/clouds.yaml
+++ /dev/null
diff --git a/nodepool/nodepool.yaml b/nodepool/nodepool.yaml
index 17cbe7d..819d500 100644
--- a/nodepool/nodepool.yaml
+++ b/nodepool/nodepool.yaml
@@ -17,7 +17,6 @@
pools:
- name: main
max-servers: 8
- #host-key-checking: False
use-internal-ip: True
labels:
- name: testlabel
diff --git a/playbooks/deploy.yaml b/playbooks/deploy.yaml
new file mode 100644
index 0000000..669d644
--- /dev/null
+++ b/playbooks/deploy.yaml
@@ -0,0 +1,133 @@
+- name: Deploy Zuul
+ hosts: localhost
+ tasks:
+ - name: Make kube directory
+ file:
+ path: "{{ ansible_user_dir }}/.kube"
+ state: directory
+ - name: Write kube config
+ template:
+ src: kubecfg.yaml.j2
+ dest: "{{ ansible_user_dir }}/.kube/config"
+
+ - name: Update Letsencrypt configuration
+ k8s:
+ state: present
+ src: ../k8s/letsencrypt.yaml
+
+ - name: Update authdaemon configuration
+ k8s:
+ state: present
+ src: ../k8s/authdaemon.yaml
+
+ - name: Update Nodepool config
+ k8s:
+ state: present
+ definition:
+ apiVersion: v1
+ kind: Secret
+ metadata:
+ namespace: zuul
+ name: nodepool-gcs
+ labels:
+ app.kubernetes.io/name: nodepool
+ app.kubernetes.io/instance: nodepool
+ app.kubernetes.io/part-of: nodepool
+ stringData:
+ clouds.yaml: ""
+ nodepool.yaml: "{{ lookup('file', '../nodepool/nodepool.yaml') }}"
+
+ - name: Update Zuul/Nodepool private key
+ k8s:
+ state: present
+ definition:
+ apiVersion: v1
+ kind: Secret
+ metadata:
+ namespace: zuul
+ name: nodepool-private-key
+ labels:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ stringData:
+ nodepool_id_rsa: "{{ zuul_deploy.nodepool_private_key }}"
+ register: zuul_config
+ # Set no_log because we are templating in the private key
+ no_log: true
+
+ - name: Update Zuul service config
+ k8s:
+ state: present
+ definition:
+ apiVersion: v1
+ kind: Secret
+ metadata:
+ namespace: zuul
+ name: zuul-config
+ labels:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ stringData:
+ main.yaml: "{{ lookup('template', '../zuul/zuul.conf') }}"
+ register: zuul_config
+ # Set no_log because we are templating passwords into the config
+ no_log: true
+
+ - name: Update Zuul tenant config
+ k8s:
+ state: present
+ definition:
+ apiVersion: v1
+ kind: Secret
+ metadata:
+ namespace: zuul
+ name: zuul-tenant-config
+ labels:
+ app.kubernetes.io/name: zuul
+ app.kubernetes.io/instance: zuul
+ app.kubernetes.io/part-of: zuul
+ stringData:
+ main.yaml: "{{ lookup('file', '../zuul/main.yaml') }}"
+ register: tenant_config
+
+ - name: Update Nodepool deployment
+ k8s:
+ state: present
+ src: ../k8s/nodepool.yaml
+
+ - name: Update Zuul deployment
+ k8s:
+ state: present
+ src: ../k8s/zuul.yaml
+
+ - name: Reconfigure Zuul
+ when: tenant_config.changed or zuul_config.changed
+ block:
+ - name: Add scheduler to inventory
+ add_host:
+ name: 'zuul-scheduler-0'
+ ansible_kubectl_namespace: zuul
+ ansible_connection: kubectl
+ - name: Wait until remote Zuul config is updated
+ delegate_to: 'zuul-scheduler-0'
+ stat:
+ path: /etc/zuul/zuul.conf
+ follow: true
+ register: remote_zuul_st
+ until: "remote_zuul_st.stat.checksum == (zuul_config.result.data['zuul.conf'] | b64decode | hash('sha1'))"
+ delay: 10
+ retries: 30
+ - name: Wait until remote tenant config is updated
+ delegate_to: 'zuul-scheduler-0'
+ stat:
+ path: /etc/zuul/tenant/main.yaml
+ follow: true
+ register: remote_tenant_st
+ until: "remote_tenant_st.stat.checksum == (tenant_config.result.data['main.yaml'] | b64decode | hash('sha1'))"
+ delay: 10
+ retries: 30
+ - name: Send reconfiguration notice to scheduler
+ delegate_to: 'zuul-scheduler-0'
+ command: zuul-scheduler full-reconfigure
diff --git a/playbooks/templates/kubecfg.yaml.j2 b/playbooks/templates/kubecfg.yaml.j2
new file mode 100644
index 0000000..8ee3d82
--- /dev/null
+++ b/playbooks/templates/kubecfg.yaml.j2
@@ -0,0 +1,20 @@
+# -*- mode: yaml -*-
+
+apiVersion: 'v1'
+kind: 'Config'
+preferences: {}
+users:
+ - name: "zuul-deploy"
+ user:
+ token: "{{ zuul_deploy.token }}"
+clusters:
+ - name: 'control-plane'
+ cluster:
+ server: "{{ zuul_deploy.server }}"
+ certificate-authority-data: "{{ zuul_deploy.cert }}"
+contexts:
+ - name: "zuul-deploy/control-plane"
+ context:
+ user: "zuul-deploy"
+ cluster: "control-plane"
+current-context: "zuul-deploy/control-plane"
diff --git a/zuul-app.yaml b/zuul-app.yaml
deleted file mode 100644
index 561e1e5..0000000
--- a/zuul-app.yaml
+++ /dev/null
@@ -1,34 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
- name: zuul
- finalizers:
- - resources-finalizer.argocd.argoproj.io
-spec:
- project: default
- source:
- repoURL: https://github.com/jeblair/zuul-helm
- path: charts/zuul
- helm:
- values: |
- zookeeper:
- hosts: zookeeper.zookeeper
- executor:
- replicas: 1
- merger:
- replicas: 1
- web:
- replicas: 1
- host: gerrit-zuul.inaugust.com
- serviceType: NodePort
- scheduler:
- tenantConfigPath: /etc/zuul/tenant/main.yaml
- tenantConfigSecret: zuul-tenant-config
- connections:
- opendev:
- name: opendev
- driver: git
- baseurl: https://opendev.org
- destination:
- namespace: zuul
- server: https://kubernetes.default.svc
diff --git a/zuul/main.yaml b/zuul/main.yaml
index 4d6631a..f5ee681 100644
--- a/zuul/main.yaml
+++ b/zuul/main.yaml
@@ -1,11 +1,12 @@
- tenant:
name: gerrit
report-build-page: true
- web-root: http://localhost:3000/
source:
opendev:
untrusted-projects:
- zuul/zuul-jobs
-# googlesource:
-# untrusted-projects:
-# - zuul/ops
+ gerrit:
+ config-projects:
+ - zuul/config
+ untrusted-projects:
+ - zuul/ops
diff --git a/zuul/zuul.conf b/zuul/zuul.conf
new file mode 100644
index 0000000..e81ad4b
--- /dev/null
+++ b/zuul/zuul.conf
@@ -0,0 +1,42 @@
+[gearman]
+server=zuul-gearman
+port=4730
+
+[zookeeper]
+hosts=zookeeper.zookeeper
+
+[gearman_server]
+start=true
+port=4730
+
+[web]
+listen_address=0.0.0.0
+root=https://gerrit-zuul.inaugust.com
+port=9000
+
+[scheduler]
+tenant_config=/etc/zuul/tenant/main.yaml
+
+[executor]
+private_key_file=/var/lib/zuul/ssh/nodepool_id_rsa
+trusted_ro_paths=/authdaemon/token
+disk_limit_per_job=500
+
+[connection "sql"]
+name=sql
+driver=sql
+dburi=mysql+pymysql://zuul:{{ zuul_deploy.db_password }}@mariadb-mariadb.mariadb/zuul
+
+[connection "opendev"]
+name=opendev
+driver=git
+baseurl=https://opendev.org
+
+[connection "gerrit"]
+name=gerrit
+driver=gerrit
+server=gerrit-review.googlesource.com
+canonical_hostname=gerrit.googlesource.com
+user=zuul
+stream_events=false
+auth_type=gcloud_service
