diff --git a/kubernetes/export-percona-helm.sh b/kubernetes/export-percona-helm.sh
new file mode 100755
index 0000000000..05f4135ab2
--- /dev/null
+++ b/kubernetes/export-percona-helm.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Assumes helm is installed in the path. Can be downloaded from
+# wget https://storage.googleapis.com/kubernetes-helm/helm-v2.12.0-linux-amd64.tar.gz
+
+K8S_DIR=$(pwd)
+BUILD_DIR=$(mktemp -d)
+
+pushd $BUILD_DIR
+
+helm fetch stable/percona-xtradb-cluster --untar
+helm template --name=gitea --set allowRootFrom=127.0.0.1,mysqlRootPassword=CHANGEMEROOTPASSWORD,xtraBackupPassword=CHANGEMEXTRABACKUP,mysqlUser=gitea,mysqlPassword=CHANGEMEPASSWORD,mysqlDatabase=gitea,persistence.enabled=true,persistence.storageClass=cinder --namespace gitea --output-dir $K8S_DIR percona-xtradb-cluster
+
+popd
+rm -rf $BUILD_DIR
+# Remove trailing whitespace
+rm -rf percona-xtradb-cluster/templates/test
+mv percona-xtradb-cluster/templates/*yaml percona-xtradb-cluster
+find percona-xtradb-cluster -type f | xargs -n1 sed -i 's/ *$//'
diff --git a/kubernetes/percona-xtradb-cluster/config-map_mysql-config.yaml b/kubernetes/percona-xtradb-cluster/config-map_mysql-config.yaml
new file mode 100644
index 0000000000..6904311b31
--- /dev/null
+++ b/kubernetes/percona-xtradb-cluster/config-map_mysql-config.yaml
@@ -0,0 +1,23 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+ name: gitea-pxc-config-files
+ labels:
+ app: gitea-pxc
+ namespace: gitea-db
+data:
+ node.cnf: |+
+ [mysqld]
+ datadir=/var/lib/mysql
+ default_storage_engine=InnoDB
+ binlog_format=ROW
+ innodb_flush_log_at_trx_commit = 0
+ innodb_flush_method = O_DIRECT
+ innodb_file_per_table = 1
+ innodb_autoinc_lock_mode=2
+ bind_address = 0.0.0.0
+ wsrep_slave_threads=2
+ wsrep_cluster_address=gcomm://
+ wsrep_provider=/usr/lib/galera3/libgalera_smm.so
+ wsrep_cluster_name=galera
+ wsrep_sst_method=xtrabackup-v2
diff --git a/kubernetes/percona-xtradb-cluster/config-map_startup-scripts.yaml b/kubernetes/percona-xtradb-cluster/config-map_startup-scripts.yaml
new file mode 100644
index 0000000000..56a05934d2
--- /dev/null
+++ b/kubernetes/percona-xtradb-cluster/config-map_startup-scripts.yaml
@@ -0,0 +1,153 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+ name: gitea-pxc-startup-scripts
+ labels:
+ app: gitea-pxc
+ namespace: gitea-db
+data:
+ entrypoint.sh: |
+ #!/bin/bash
+ set -e
+
+ if [[ -n "${DEBUG}" ]]; then
+ set -x
+ fi
+
+ . /startup-scripts/functions.sh
+
+ ipaddr=$(hostname -i | awk ' { print $1 } ')
+ hostname=$(hostname)
+ echo "I AM $hostname - $ipaddr"
+
+ # if command starts with an option, prepend mysqld
+ if [ "${1:0:1}" = '-' ]; then
+ CMDARG="$@"
+ fi
+
+ cluster_join=$(resolveip -s "${K8S_SERVICE_NAME}" || echo "")
+ if [[ -z "${cluster_join}" ]]; then
+ echo "I am the Primary Node"
+ init_mysql
+ write_password_file
+ exec mysqld --user=mysql --wsrep_cluster_name=$SHORT_CLUSTER_NAME --wsrep_node_name=$hostname \
+ --wsrep_cluster_address=gcomm:// --wsrep_sst_method=xtrabackup-v2 \
+ --wsrep_sst_auth="xtrabackup:$XTRABACKUP_PASSWORD" \
+ --wsrep_node_address="$ipaddr" $CMDARG
+ else
+ echo "I am not the Primary Node"
+ chown -R mysql:mysql /var/lib/mysql
+ touch /var/log/mysqld.log
+ chown mysql:mysql /var/log/mysqld.log
+ write_password_file
+ exec mysqld --user=mysql --wsrep_cluster_name=$SHORT_CLUSTER_NAME --wsrep_node_name=$hostname \
+ --wsrep_cluster_address="gcomm://$cluster_join" --wsrep_sst_method=xtrabackup-v2 \
+ --wsrep_sst_auth="xtrabackup:$XTRABACKUP_PASSWORD" \
+ --wsrep_node_address="$ipaddr" $CMDARG
+ fi
+
+ functions.sh: |
+ #!/bin/bash
+
+ write_password_file() {
+ if [[ -n "${MYSQL_ROOT_PASSWORD}" ]]; then
+ cat <<EOF > /root/.my.cnf
+ [client]
+ user=root
+ password=${MYSQL_ROOT_PASSWORD}
+ EOF
+ fi
+ }
+
+ init_mysql() {
+ DATADIR=/var/lib/mysql
+ # if we have CLUSTER_JOIN - then we do not need to perform datadir initialize
+ # the data will be copied from another node
+ if [ ! -e "$DATADIR/mysql" ]; then
+ if [ -z "$MYSQL_ROOT_PASSWORD" -a -z "$MYSQL_ALLOW_EMPTY_PASSWORD" -a -z "$MYSQL_RANDOM_ROOT_PASSWORD" -a -z "$MYSQL_ROOT_PASSWORD_FILE" ]; then
+ echo >&2 'error: database is uninitialized and password option is not specified '
+ echo >&2 ' You need to specify one of MYSQL_ROOT_PASSWORD, MYSQL_ROOT_PASSWORD_FILE, MYSQL_ALLOW_EMPTY_PASSWORD or MYSQL_RANDOM_ROOT_PASSWORD'
+ exit 1
+ fi
+
+ if [ ! -z "$MYSQL_ROOT_PASSWORD_FILE" -a -z "$MYSQL_ROOT_PASSWORD" ]; then
+ MYSQL_ROOT_PASSWORD=$(cat $MYSQL_ROOT_PASSWORD_FILE)
+ fi
+ mkdir -p "$DATADIR"
+
+ echo "Running --initialize-insecure on $DATADIR"
+ ls -lah $DATADIR
+ mysqld --initialize-insecure
+ chown -R mysql:mysql "$DATADIR"
+ chown mysql:mysql /var/log/mysqld.log
+ echo 'Finished --initialize-insecure'
+
+ mysqld --user=mysql --datadir="$DATADIR" --skip-networking &
+ pid="$!"
+
+ mysql=( mysql --protocol=socket -uroot )
+
+ for i in {30..0}; do
+ if echo 'SELECT 1' | "${mysql[@]}" &> /dev/null; then
+ break
+ fi
+ echo 'MySQL init process in progress...'
+ sleep 1
+ done
+ if [ "$i" = 0 ]; then
+ echo >&2 'MySQL init process failed.'
+ exit 1
+ fi
+
+ # sed is for https://bugs.mysql.com/bug.php?id=20545
+ mysql_tzinfo_to_sql /usr/share/zoneinfo | sed 's/Local time zone must be set--see zic manual page/FCTY/' | "${mysql[@]}" mysql
+ "${mysql[@]}" <<-EOSQL
+ -- What's done in this file shouldn't be replicated
+ -- or products like mysql-fabric won't work
+ SET @@SESSION.SQL_LOG_BIN=0;
+ CREATE USER 'root'@'${ALLOW_ROOT_FROM}' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' ;
+ GRANT ALL ON *.* TO 'root'@'${ALLOW_ROOT_FROM}' WITH GRANT OPTION ;
+ ALTER USER 'root'@'localhost' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}';
+ GRANT ALL ON *.* TO 'root'@'localhost' WITH GRANT OPTION ;
+ CREATE USER 'xtrabackup'@'localhost' IDENTIFIED BY '$XTRABACKUP_PASSWORD';
+ GRANT RELOAD,PROCESS,LOCK TABLES,REPLICATION CLIENT ON *.* TO 'xtrabackup'@'localhost';
+ GRANT REPLICATION CLIENT ON *.* TO monitor@'%' IDENTIFIED BY 'monitor';
+ GRANT PROCESS ON *.* TO monitor@localhost IDENTIFIED BY 'monitor';
+ DROP DATABASE IF EXISTS test ;
+ FLUSH PRIVILEGES ;
+ EOSQL
+ if [ ! -z "$MYSQL_ROOT_PASSWORD" ]; then
+ mysql+=( -p"${MYSQL_ROOT_PASSWORD}" )
+ fi
+
+ if [ "$MYSQL_DATABASE" ]; then
+ echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" | "${mysql[@]}"
+ mysql+=( "$MYSQL_DATABASE" )
+ fi
+
+ if [ "$MYSQL_USER" -a "$MYSQL_PASSWORD" ]; then
+ echo "CREATE USER '"$MYSQL_USER"'@'%' IDENTIFIED BY '"$MYSQL_PASSWORD"' ;" | "${mysql[@]}"
+
+ if [ "$MYSQL_DATABASE" ]; then
+ echo "GRANT ALL ON \`"$MYSQL_DATABASE"\`.* TO '"$MYSQL_USER"'@'%' ;" | "${mysql[@]}"
+ fi
+
+ echo 'FLUSH PRIVILEGES ;' | "${mysql[@]}"
+ fi
+
+ if [ ! -z "$MYSQL_ONETIME_PASSWORD" ]; then
+ "${mysql[@]}" <<-EOSQL
+ ALTER USER 'root'@'%' PASSWORD EXPIRE;
+ EOSQL
+ fi
+ if ! kill -s TERM "$pid" || ! wait "$pid"; then
+ echo >&2 'MySQL init process failed.'
+ exit 1
+ fi
+
+ echo
+ echo 'MySQL init process done. Ready for start up.'
+ echo
+ fi
+
+ }
diff --git a/kubernetes/percona-xtradb-cluster/gitea-db-namespace.yaml b/kubernetes/percona-xtradb-cluster/gitea-db-namespace.yaml
new file mode 100644
index 0000000000..d74dfda9c9
--- /dev/null
+++ b/kubernetes/percona-xtradb-cluster/gitea-db-namespace.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: gitea-db
diff --git a/kubernetes/percona-xtradb-cluster/pxc-playbook.yaml b/kubernetes/percona-xtradb-cluster/pxc-playbook.yaml
new file mode 100644
index 0000000000..44e3638d7e
--- /dev/null
+++ b/kubernetes/percona-xtradb-cluster/pxc-playbook.yaml
@@ -0,0 +1,46 @@
+- hosts: localhost
+ tasks:
+ - name: Set up cinder storage class
+ k8s:
+ state: present
+ definition: "{{ lookup('file', 'storage-class.yaml') | from_yaml }}"
+
+ - name: Set up gitea-db namespace
+ k8s:
+ state: present
+ definition: "{{ lookup('template', 'gitea-db-namespace.yaml') | from_yaml }}"
+
+ - name: Set up gitea-db secrets
+ k8s:
+ state: present
+ definition: "{{ lookup('template', 'secrets.yaml') | from_yaml }}"
+
+ - name: Set up gitea-db mysql config configmap
+ k8s:
+ state: present
+ definition: "{{ lookup('file', 'config-map_mysql-config.yaml') | from_yaml }}"
+
+ - name: Set up gitea-db startup scripts configmap
+ k8s:
+ state: present
+ definition: "{{ lookup('file', 'config-map_startup-scripts.yaml') | from_yaml }}"
+
+ - name: Set up gitea-db xtradb cluster statefulset
+ k8s:
+ state: present
+ definition: "{{ lookup('file', 'statefulset.yaml') | from_yaml }}"
+
+ - name: Set up gitea-db metrics service
+ k8s:
+ state: present
+ definition: "{{ lookup('file', 'service-metrics.yaml') | from_yaml }}"
+
+ - name: Set up gitea-db database service
+ k8s:
+ state: present
+ definition: "{{ lookup('file', 'service-percona.yaml') | from_yaml }}"
+
+ - name: Set up gitea-db galera replication service
+ k8s:
+ state: present
+ definition: "{{ lookup('file', 'service-repl.yaml') | from_yaml }}"
diff --git a/kubernetes/percona-xtradb-cluster/secrets.yaml b/kubernetes/percona-xtradb-cluster/secrets.yaml
new file mode 100644
index 0000000000..2b9d5d59fd
--- /dev/null
+++ b/kubernetes/percona-xtradb-cluster/secrets.yaml
@@ -0,0 +1,13 @@
+# Source: percona-xtradb-cluster/templates/secrets.yaml
+apiVersion: v1
+kind: Secret
+metadata:
+ name: gitea-pxc
+ labels:
+ app: gitea-pxc
+ namespace: gitea-db
+type: Opaque
+stringData:
+ mysql-root-password: {{ gitea_root_db_password }}
+ mysql-password: {{ gitea_db_password }}
+ xtrabackup-password: {{ gitea_xtrabackup_password }}
diff --git a/kubernetes/percona-xtradb-cluster/service-metrics.yaml b/kubernetes/percona-xtradb-cluster/service-metrics.yaml
new file mode 100644
index 0000000000..4978c71b41
--- /dev/null
+++ b/kubernetes/percona-xtradb-cluster/service-metrics.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: "gitea-pxc-metrics"
+ labels:
+ app: gitea-pxc
+ namespace: gitea-db
+spec:
+ clusterIP: None
+ ports:
+ - port: 9104
+ selector:
+ app: gitea-pxc
diff --git a/kubernetes/percona-xtradb-cluster/service-percona.yaml b/kubernetes/percona-xtradb-cluster/service-percona.yaml
new file mode 100644
index 0000000000..36e3558a00
--- /dev/null
+++ b/kubernetes/percona-xtradb-cluster/service-percona.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: gitea-pxc
+ labels:
+ app: gitea-pxc
+ namespace: gitea-db
+spec:
+ ports:
+ - name: mysql
+ port: 3306
+ targetPort: mysql
+ selector:
+ app: gitea-pxc
diff --git a/kubernetes/percona-xtradb-cluster/service-repl.yaml b/kubernetes/percona-xtradb-cluster/service-repl.yaml
new file mode 100644
index 0000000000..8f2d0b3a94
--- /dev/null
+++ b/kubernetes/percona-xtradb-cluster/service-repl.yaml
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: "gitea-pxc-repl"
+ labels:
+ app: gitea-pxc
+ namespace: gitea-db
+spec:
+ clusterIP: None
+ ports:
+ - name: galera
+ port: 4567
+ - name: state-xfer
+ port: 4568
+ - name: state-snap
+ port: 4444
+ selector:
+ app: gitea-pxc
diff --git a/kubernetes/percona-xtradb-cluster/statefulset.yaml b/kubernetes/percona-xtradb-cluster/statefulset.yaml
new file mode 100644
index 0000000000..2f44f33c4a
--- /dev/null
+++ b/kubernetes/percona-xtradb-cluster/statefulset.yaml
@@ -0,0 +1,131 @@
+apiVersion: apps/v1beta2
+kind: StatefulSet
+metadata:
+ name: gitea-pxc
+ labels:
+ app: gitea-pxc
+ namespace: gitea-db
+spec:
+ replicas: 3
+ selector:
+ matchLabels:
+ app: gitea-pxc
+ serviceName: gitea-pxc
+ template:
+ metadata:
+ labels:
+ app: gitea-pxc
+ spec:
+ initContainers:
+ - name: "remove-lost-found"
+ image: "busybox:1.25.0"
+ imagePullPolicy: IfNotPresent
+ command:
+ - "rm"
+ - "-fr"
+ - "/var/lib/mysql/lost+found"
+ volumeMounts:
+ - name: mysql-data
+ mountPath: /var/lib/mysql
+ containers:
+ - name: database
+ image: "percona/percona-xtradb-cluster:5.7.19"
+ imagePullPolicy: "IfNotPresent"
+ command:
+ - "/bin/bash"
+ - "/startup-scripts/entrypoint.sh"
+ resources:
+ null
+
+ env:
+ - name: MYSQL_ROOT_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: gitea-pxc
+ key: mysql-root-password
+ - name: MYSQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: gitea-pxc
+ key: mysql-password
+ - name: XTRABACKUP_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: gitea-pxc
+ key: xtrabackup-password
+ - name: MYSQL_USER
+ value: "gitea"
+ - name: MYSQL_DATABASE
+ value: "gitea"
+ - name: ALLOW_ROOT_FROM
+ value: "127.0.0.1"
+ - name: CLUSTER_NAME
+ value: gitea-pxc
+ - name: SHORT_CLUSTER_NAME
+ value: gitea-pxc
+ - name: K8S_SERVICE_NAME
+ value: gitea-pxc-repl
+ - name: DEBUG
+ value: "true"
+ ports:
+ - name: mysql
+ containerPort: 3306
+ - name: galera-repl
+ containerPort: 4567
+ - name: state-transfer
+ containerPort: 4568
+ - name: state-snapshot
+ containerPort: 4444
+ livenessProbe:
+ exec:
+ command: ["mysqladmin","ping"]
+ initialDelaySeconds: 30
+ timeoutSeconds: 2
+ readinessProbe:
+ exec:
+ command: ["mysql", "-h", "127.0.0.1", "-e", "SELECT 1"]
+ initialDelaySeconds: 30
+ timeoutSeconds: 2
+ volumeMounts:
+ - name: mysql-data
+ mountPath: /var/lib/mysql
+ - name: mysql-startup-scripts
+ mountPath: /startup-scripts
+ - name: mysql-config-files
+ mountPath: /etc/mysql/conf.d
+ - name: slash-root
+ mountPath: /root
+ - name: var-log
+ mountPath: /var/log
+
+ - name: "logs"
+ image: "busybox:1.25.0"
+ imagePullPolicy: IfNotPresent
+ command:
+ - "tail"
+ - "-f"
+ - "/var/log/mysqld.log"
+ volumeMounts:
+ - name: var-log
+ mountPath: /var/log
+ volumes:
+ - name: slash-root
+ emptyDir: {}
+ - name: var-log
+ emptyDir: {}
+ - name: mysql-config-files
+ configMap:
+ name: gitea-pxc-config-files
+ - name: mysql-startup-scripts
+ configMap:
+ name: gitea-pxc-startup-scripts
+ volumeClaimTemplates:
+ - metadata:
+ name: mysql-data
+ namespace: gitea-db
+ spec:
+ accessModes: ["ReadWriteOnce"]
+ storageClassName: "cinder"
+ resources:
+ requests:
+ storage: "24Gi"
diff --git a/kubernetes/percona-xtradb-cluster/storage-class.yaml b/kubernetes/percona-xtradb-cluster/storage-class.yaml
new file mode 100644
index 0000000000..a6e0b4d60f
--- /dev/null
+++ b/kubernetes/percona-xtradb-cluster/storage-class.yaml
@@ -0,0 +1,11 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+ name: cinder
+ labels:
+ kubernetes.io/cluster-service: "true"
+ addonmanager.kubernetes.io/mode: EnsureExists
+provisioner: kubernetes.io/cinder
+parameters:
+ type: rbd
+ availability: nova
diff --git a/kubernetes/rook/cluster.yaml b/kubernetes/rook/cluster.yaml
new file mode 100644
index 0000000000..fc4a04a271
--- /dev/null
+++ b/kubernetes/rook/cluster.yaml
@@ -0,0 +1,260 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: rook-ceph
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: rook-ceph-osd
+ namespace: rook-ceph
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: rook-ceph-mgr
+ namespace: rook-ceph
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: rook-ceph-osd
+ namespace: rook-ceph
+rules:
+- apiGroups: [""]
+ resources: ["configmaps"]
+ verbs: [ "get", "list", "watch", "create", "update", "delete" ]
+---
+# Aspects of ceph-mgr that require access to the system namespace
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: rook-ceph-mgr-system
+ namespace: rook-ceph
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - configmaps
+ verbs:
+ - get
+ - list
+ - watch
+---
+# Aspects of ceph-mgr that operate within the cluster's namespace
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: rook-ceph-mgr
+ namespace: rook-ceph
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - pods
+ - services
+ verbs:
+ - get
+ - list
+ - watch
+- apiGroups:
+ - batch
+ resources:
+ - jobs
+ verbs:
+ - get
+ - list
+ - watch
+ - create
+ - update
+ - delete
+- apiGroups:
+ - ceph.rook.io
+ resources:
+ - "*"
+ verbs:
+ - "*"
+---
+# Allow the operator to create resources in this cluster's namespace
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: rook-ceph-cluster-mgmt
+ namespace: rook-ceph
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: rook-ceph-cluster-mgmt
+subjects:
+- kind: ServiceAccount
+ name: rook-ceph-system
+ namespace: rook-ceph-system
+---
+# Allow the osd pods in this namespace to work with configmaps
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: rook-ceph-osd
+ namespace: rook-ceph
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: rook-ceph-osd
+subjects:
+- kind: ServiceAccount
+ name: rook-ceph-osd
+ namespace: rook-ceph
+---
+# Allow the ceph mgr to access the cluster-specific resources necessary for the mgr modules
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: rook-ceph-mgr
+ namespace: rook-ceph
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: rook-ceph-mgr
+subjects:
+- kind: ServiceAccount
+ name: rook-ceph-mgr
+ namespace: rook-ceph
+---
+# Allow the ceph mgr to access the rook system resources necessary for the mgr modules
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: rook-ceph-mgr-system
+ namespace: rook-ceph-system
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: rook-ceph-mgr-system
+subjects:
+- kind: ServiceAccount
+ name: rook-ceph-mgr
+ namespace: rook-ceph
+---
+# Allow the ceph mgr to access cluster-wide resources necessary for the mgr modules
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: rook-ceph-mgr-cluster
+ namespace: rook-ceph
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: rook-ceph-mgr-cluster
+subjects:
+- kind: ServiceAccount
+ name: rook-ceph-mgr
+ namespace: rook-ceph
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+ name: rook-ceph
+ namespace: rook-ceph
+spec:
+ cephVersion:
+ # The container image used to launch the Ceph daemon pods (mon, mgr, osd, mds, rgw).
+ # v12 is luminous, v13 is mimic, and v14 is nautilus.
+ # RECOMMENDATION: In production, use a specific version tag instead of the general v13 flag, which pulls the latest release and could result in different
+ # versions running within the cluster. See tags available at https://hub.docker.com/r/ceph/ceph/tags/.
+ image: ceph/ceph:v13.2.2-20181206
+ # Whether to allow unsupported versions of Ceph. Currently only luminous and mimic are supported.
+ # After nautilus is released, Rook will be updated to support nautilus.
+ # Do not set to true in production.
+ allowUnsupported: false
+ # The path on the host where configuration files will be persisted. If not specified, a kubernetes emptyDir will be created (not recommended).
+ # Important: if you reinstall the cluster, make sure you delete this directory from each host or else the mons will fail to start on the new cluster.
+ # In Minikube, the '/data' directory is configured to persist across reboots. Use "/data/rook" in Minikube environment.
+ dataDirHostPath: /var/lib/rook
+ # set the amount of mons to be started
+ mon:
+ count: 3
+ allowMultiplePerNode: true
+ # enable the ceph dashboard for viewing cluster status
+ dashboard:
+ enabled: true
+ # serve the dashboard under a subpath (useful when you are accessing the dashboard via a reverse proxy)
+ # urlPrefix: /ceph-dashboard
+ network:
+ # toggle to use hostNetwork
+ hostNetwork: false
+ rbdMirroring:
+ # The number of daemons that will perform the rbd mirroring.
+ # rbd mirroring must be configured with "rbd mirror" from the rook toolbox.
+ workers: 0
+ # To control where various services will be scheduled by kubernetes, use the placement configuration sections below.
+ # The example under 'all' would have all services scheduled on kubernetes nodes labeled with 'role=storage-node' and
+ # tolerate taints with a key of 'storage-node'.
+# placement:
+# all:
+# nodeAffinity:
+# requiredDuringSchedulingIgnoredDuringExecution:
+# nodeSelectorTerms:
+# - matchExpressions:
+# - key: role
+# operator: In
+# values:
+# - storage-node
+# podAffinity:
+# podAntiAffinity:
+# tolerations:
+# - key: storage-node
+# operator: Exists
+# The above placement information can also be specified for mon, osd, and mgr components
+# mon:
+# osd:
+# mgr:
+ resources:
+# The requests and limits set here, allow the mgr pod to use half of one CPU core and 1 gigabyte of memory
+# mgr:
+# limits:
+# cpu: "500m"
+# memory: "1024Mi"
+# requests:
+# cpu: "500m"
+# memory: "1024Mi"
+# The above example requests/limits can also be added to the mon and osd components
+# mon:
+# osd:
+ storage: # cluster level storage configuration and selection
+ useAllNodes: true
+ useAllDevices: true
+ deviceFilter:
+ location:
+ config:
+ # The default and recommended storeType is dynamically set to bluestore for devices and filestore for directories.
+ # Set the storeType explicitly only if it is required not to use the default.
+ storeType: bluestore
+ databaseSizeMB: "1024" # this value can be removed for environments with normal sized disks (100 GB or larger)
+ journalSizeMB: "1024" # this value can be removed for environments with normal sized disks (20 GB or larger)
+ osdsPerDevice: "1" # this value can be overridden at the node or device level
+# Cluster level list of directories to use for storage. These values will be set for all nodes that have no `directories` set.
+# directories:
+# - path: /var/lib/rook/storage-dir
+# Individual nodes and their config can be specified as well, but 'useAllNodes' above must be set to false. Then, only the named
+# nodes below will be used as storage resources. Each node's 'name' field should match their 'kubernetes.io/hostname' label.
+# nodes:
+# - name: "172.17.4.101"
+# directories: # specific directories to use for storage can be specified for each node
+# - path: "/rook/storage-dir"
+# resources:
+# limits:
+# cpu: "500m"
+# memory: "1024Mi"
+# requests:
+# cpu: "500m"
+# memory: "1024Mi"
+# - name: "172.17.4.201"
+# devices: # specific devices to use for storage can be specified for each node
+# - name: "sdb"
+# - name: "nvme01" # multiple osds can be created on high performance devices
+# config:
+# osdsPerDevice: "5"
+# config: # configuration can be specified at the node level which overrides the cluster level config
+# storeType: filestore
+# - name: "172.17.4.301"
+# deviceFilter: "^sd."
diff --git a/kubernetes/rook/filesystem.yaml b/kubernetes/rook/filesystem.yaml
new file mode 100644
index 0000000000..6b2fa2adf7
--- /dev/null
+++ b/kubernetes/rook/filesystem.yaml
@@ -0,0 +1,46 @@
+apiVersion: ceph.rook.io/v1
+kind: CephFilesystem
+metadata:
+ name: myfs
+ namespace: rook-ceph
+spec:
+ # The metadata pool spec
+ metadataPool:
+ replicated:
+ # Increase the replication size if you have more than one osd
+ size: 3
+ # The list of data pool specs
+ dataPools:
+ - failureDomain: osd
+ replicated:
+ size: 3
+ # The metadata service (mds) configuration
+ metadataServer:
+ # The number of active MDS instances
+ activeCount: 1
+ # Whether each active MDS instance will have an active standby with a warm metadata cache for faster failover.
+ # If false, standbys will be available, but will not have a warm cache.
+ activeStandby: true
+ # The affinity rules to apply to the mds deployment
+ placement:
+ # nodeAffinity:
+ # requiredDuringSchedulingIgnoredDuringExecution:
+ # nodeSelectorTerms:
+ # - matchExpressions:
+ # - key: role
+ # operator: In
+ # values:
+ # - mds-node
+ # tolerations:
+ # - key: mds-node
+ # operator: Exists
+ # podAffinity:
+ # podAntiAffinity:
+ resources:
+ # The requests and limits set here, allow the filesystem MDS Pod(s) to use half of one CPU core and 1 gigabyte of memory
+ # limits:
+ # cpu: "500m"
+ # memory: "1024Mi"
+ # requests:
+ # cpu: "500m"
+ # memory: "1024Mi"
diff --git a/kubernetes/rook/operator.yaml b/kubernetes/rook/operator.yaml
new file mode 100644
index 0000000000..987c898ab3
--- /dev/null
+++ b/kubernetes/rook/operator.yaml
@@ -0,0 +1,474 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: rook-ceph-system
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ name: cephclusters.ceph.rook.io
+spec:
+ group: ceph.rook.io
+ names:
+ kind: CephCluster
+ listKind: CephClusterList
+ plural: cephclusters
+ singular: cephcluster
+ scope: Namespaced
+ version: v1
+ validation:
+ openAPIV3Schema:
+ properties:
+ spec:
+ properties:
+ cephVersion:
+ properties:
+ allowUnsupported:
+ type: boolean
+ image:
+ type: string
+ name:
+ pattern: ^(luminous|mimic|nautilus)$
+ type: string
+ dashboard:
+ properties:
+ enabled:
+ type: boolean
+ urlPrefix:
+ type: string
+ dataDirHostPath:
+ pattern: ^/(\S+)
+ type: string
+ mon:
+ properties:
+ allowMultiplePerNode:
+ type: boolean
+ count:
+ maximum: 9
+ minimum: 1
+ type: integer
+ required:
+ - count
+ network:
+ properties:
+ hostNetwork:
+ type: boolean
+ storage:
+ properties:
+ nodes:
+ items: {}
+ type: array
+ useAllDevices: {}
+ useAllNodes:
+ type: boolean
+ required:
+ - mon
+ additionalPrinterColumns:
+ - name: DataDirHostPath
+ type: string
+ description: Directory used on the K8s nodes
+ JSONPath: .spec.dataDirHostPath
+ - name: MonCount
+ type: string
+ description: Number of MONs
+ JSONPath: .spec.mon.count
+ - name: Age
+ type: date
+ JSONPath: .metadata.creationTimestamp
+ - name: State
+ type: string
+ description: Current State
+ JSONPath: .status.state
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ name: cephfilesystems.ceph.rook.io
+spec:
+ group: ceph.rook.io
+ names:
+ kind: CephFilesystem
+ listKind: CephFilesystemList
+ plural: cephfilesystems
+ singular: cephfilesystem
+ scope: Namespaced
+ version: v1
+ additionalPrinterColumns:
+ - name: MdsCount
+ type: string
+ description: Number of MDSs
+ JSONPath: .spec.metadataServer.activeCount
+ - name: Age
+ type: date
+ JSONPath: .metadata.creationTimestamp
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ name: cephobjectstores.ceph.rook.io
+spec:
+ group: ceph.rook.io
+ names:
+ kind: CephObjectStore
+ listKind: CephObjectStoreList
+ plural: cephobjectstores
+ singular: cephobjectstore
+ scope: Namespaced
+ version: v1
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ name: cephobjectstoreusers.ceph.rook.io
+spec:
+ group: ceph.rook.io
+ names:
+ kind: CephObjectStoreUser
+ listKind: CephObjectStoreUserList
+ plural: cephobjectstoreusers
+ singular: cephobjectstoreuser
+ scope: Namespaced
+ version: v1
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ name: cephblockpools.ceph.rook.io
+spec:
+ group: ceph.rook.io
+ names:
+ kind: CephBlockPool
+ listKind: CephBlockPoolList
+ plural: cephblockpools
+ singular: cephblockpool
+ scope: Namespaced
+ version: v1
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ name: volumes.rook.io
+spec:
+ group: rook.io
+ names:
+ kind: Volume
+ listKind: VolumeList
+ plural: volumes
+ singular: volume
+ shortNames:
+ - rv
+ scope: Namespaced
+ version: v1alpha2
+---
+# The cluster role for managing all the cluster-specific resources in a namespace
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+ name: rook-ceph-cluster-mgmt
+ labels:
+ operator: rook
+ storage-backend: ceph
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - secrets
+ - pods
+ - pods/log
+ - services
+ - configmaps
+ verbs:
+ - get
+ - list
+ - watch
+ - patch
+ - create
+ - update
+ - delete
+- apiGroups:
+ - extensions
+ resources:
+ - deployments
+ - daemonsets
+ - replicasets
+ verbs:
+ - get
+ - list
+ - watch
+ - create
+ - update
+ - delete
+---
+# The role for the operator to manage resources in the system namespace
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+ name: rook-ceph-system
+ namespace: rook-ceph-system
+ labels:
+ operator: rook
+ storage-backend: ceph
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - pods
+ - configmaps
+ verbs:
+ - get
+ - list
+ - watch
+ - patch
+ - create
+ - update
+ - delete
+- apiGroups:
+ - extensions
+ resources:
+ - daemonsets
+ verbs:
+ - get
+ - list
+ - watch
+ - create
+ - update
+ - delete
+---
+# The cluster role for managing the Rook CRDs
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+ name: rook-ceph-global
+ labels:
+ operator: rook
+ storage-backend: ceph
+rules:
+- apiGroups:
+ - ""
+ resources:
+ # Pod access is needed for fencing
+ - pods
+ # Node access is needed for determining nodes where mons should run
+ - nodes
+ - nodes/proxy
+ verbs:
+ - get
+ - list
+ - watch
+- apiGroups:
+ - ""
+ resources:
+ - events
+ # PVs and PVCs are managed by the Rook provisioner
+ - persistentvolumes
+ - persistentvolumeclaims
+ verbs:
+ - get
+ - list
+ - watch
+ - patch
+ - create
+ - update
+ - delete
+- apiGroups:
+ - storage.k8s.io
+ resources:
+ - storageclasses
+ verbs:
+ - get
+ - list
+ - watch
+- apiGroups:
+ - batch
+ resources:
+ - jobs
+ verbs:
+ - get
+ - list
+ - watch
+ - create
+ - update
+ - delete
+- apiGroups:
+ - ceph.rook.io
+ resources:
+ - "*"
+ verbs:
+ - "*"
+- apiGroups:
+ - rook.io
+ resources:
+ - "*"
+ verbs:
+ - "*"
+---
+# Aspects of ceph-mgr that require cluster-wide access
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: rook-ceph-mgr-cluster
+ labels:
+ operator: rook
+ storage-backend: ceph
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - configmaps
+ - nodes
+ - nodes/proxy
+ verbs:
+ - get
+ - list
+ - watch
+---
+# The rook system service account used by the operator, agent, and discovery pods
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: rook-ceph-system
+ namespace: rook-ceph-system
+ labels:
+ operator: rook
+ storage-backend: ceph
+---
+# Grant the operator, agent, and discovery agents access to resources in the rook-ceph-system namespace
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: rook-ceph-system
+ namespace: rook-ceph-system
+ labels:
+ operator: rook
+ storage-backend: ceph
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: rook-ceph-system
+subjects:
+- kind: ServiceAccount
+ name: rook-ceph-system
+ namespace: rook-ceph-system
+---
+# Grant the rook system daemons cluster-wide access to manage the Rook CRDs, PVCs, and storage classes
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: rook-ceph-global
+ namespace: rook-ceph-system
+ labels:
+ operator: rook
+ storage-backend: ceph
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: rook-ceph-global
+subjects:
+- kind: ServiceAccount
+ name: rook-ceph-system
+ namespace: rook-ceph-system
+---
+# The deployment for the rook operator
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+ name: rook-ceph-operator
+ namespace: rook-ceph-system
+ labels:
+ operator: rook
+ storage-backend: ceph
+spec:
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: rook-ceph-operator
+ spec:
+ serviceAccountName: rook-ceph-system
+ containers:
+ - name: rook-ceph-operator
+ image: rook/ceph:v0.9.0
+ args: ["ceph", "operator"]
+ volumeMounts:
+ - mountPath: /var/lib/rook
+ name: rook-config
+ - mountPath: /etc/ceph
+ name: default-config-dir
+ env:
+ # To disable RBAC, uncomment the following:
+ # - name: RBAC_ENABLED
+ # value: "false"
+ # Rook Agent toleration. Will tolerate all taints with all keys.
+ # Choose between NoSchedule, PreferNoSchedule and NoExecute:
+ # - name: AGENT_TOLERATION
+ # value: "NoSchedule"
+ # (Optional) Rook Agent toleration key. Set this to the key of the taint you want to tolerate
+ # - name: AGENT_TOLERATION_KEY
+ # value: "<KeyOfTheTaintToTolerate>"
+ # (Optional) Rook Agent mount security mode. Can by `Any` or `Restricted`.
+ # `Any` uses Ceph admin credentials by default/fallback.
+ # For using `Restricted` you must have a Ceph secret in each namespace storage should be consumed from and
+ # set `mountUser` to the Ceph user, `mountSecret` to the Kubernetes secret name.
+ # to the namespace in which the `mountSecret` Kubernetes secret namespace.
+ # - name: AGENT_MOUNT_SECURITY_MODE
+ # value: "Any"
+ # Set the path where the Rook agent can find the flex volumes
+ # - name: FLEXVOLUME_DIR_PATH
+ # value: /var/lib/kubelet/volumeplugins
+ # Set the path where kernel modules can be found
+ # - name: LIB_MODULES_DIR_PATH
+ # value: "<PathToLibModules>"
+ # Mount any extra directories into the agent container
+ # - name: AGENT_MOUNTS
+ # value: "rootfs=/:/rootfs,varlibkubelet=/var/lib/kubelet:/var/lib/kubelet"
+ # Rook Discover toleration. Will tolerate all taints with all keys.
+ # Choose between NoSchedule, PreferNoSchedule and NoExecute:
+ # - name: DISCOVER_TOLERATION
+ # value: "NoSchedule"
+ # (Optional) Rook Discover toleration key. Set this to the key of the taint you want to tolerate
+ # - name: DISCOVER_TOLERATION_KEY
+ # value: "<KeyOfTheTaintToTolerate>"
+ # Allow rook to create multiple file systems. Note: This is considered
+ # an experimental feature in Ceph as described at
+ # http://docs.ceph.com/docs/master/cephfs/experimental-features/#multiple-filesystems-within-a-ceph-cluster
+ # which might cause mons to crash as seen in https://github.com/rook/rook/issues/1027
+ - name: ROOK_ALLOW_MULTIPLE_FILESYSTEMS
+ value: "false"
+ # The logging level for the operator: INFO | DEBUG
+ - name: ROOK_LOG_LEVEL
+ value: "INFO"
+ # The interval to check if every mon is in the quorum.
+ - name: ROOK_MON_HEALTHCHECK_INTERVAL
+ value: "45s"
+ # The duration to wait before trying to failover or remove/replace the
+ # current mon with a new mon (useful for compensating flapping network).
+ - name: ROOK_MON_OUT_TIMEOUT
+ value: "300s"
+ # The duration between discovering devices in the rook-discover daemonset.
+ - name: ROOK_DISCOVER_DEVICES_INTERVAL
+ value: "60m"
+ # Whether to start pods as privileged that mount a host path, which includes the Ceph mon and osd pods.
+ # This is necessary to workaround the anyuid issues when running on OpenShift.
+ # For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641
+ - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED
+ value: "false"
+ # The name of the node to pass with the downward API
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ # The pod name to pass with the downward API
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ # The pod namespace to pass with the downward API
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ volumes:
+ - name: rook-config
+ emptyDir: {}
+ - name: default-config-dir
+ emptyDir: {}
diff --git a/kubernetes/rook/rook-playbook.yaml b/kubernetes/rook/rook-playbook.yaml
new file mode 100644
index 0000000000..955bfab55b
--- /dev/null
+++ b/kubernetes/rook/rook-playbook.yaml
@@ -0,0 +1,14 @@
+- hosts: localhost
+ tasks:
+
+ # Using kubectl rather than k8s module because the
+ # k8s module only allows single document files and
+ # the majority of these files are pretty straight
+ # copies from upstream rook, so reorganizing them
+ # into single document files is lame.
+ - name: Set up cinder storage class
+ command: |
+ kubectl apply -f rook-operator.yaml
+ kubectl apply -f rook-cluster.yaml
+ kubectl apply -f rook-toolbox.yaml
+ kubectl apply -f rook-filesystem.yaml
diff --git a/kubernetes/rook/toolbox.yaml b/kubernetes/rook/toolbox.yaml
new file mode 100644
index 0000000000..4a0ca8fd37
--- /dev/null
+++ b/kubernetes/rook/toolbox.yaml
@@ -0,0 +1,59 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: rook-ceph-tools
+ namespace: rook-ceph
+ labels:
+ app: rook-ceph-tools
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: rook-ceph-tools
+ template:
+ metadata:
+ labels:
+ app: rook-ceph-tools
+ spec:
+ dnsPolicy: ClusterFirstWithHostNet
+ containers:
+ - name: rook-ceph-tools
+ image: rook/ceph:v0.9.0
+ command: ["/tini"]
+ args: ["-g", "--", "/usr/local/bin/toolbox.sh"]
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: ROOK_ADMIN_SECRET
+ valueFrom:
+ secretKeyRef:
+ name: rook-ceph-mon
+ key: admin-secret
+ securityContext:
+ privileged: true
+ volumeMounts:
+ - mountPath: /dev
+ name: dev
+ - mountPath: /sys/bus
+ name: sysbus
+ - mountPath: /lib/modules
+ name: libmodules
+ - name: mon-endpoint-volume
+ mountPath: /etc/rook
+ # if hostNetwork: false, the "rbd map" command hangs, see https://github.com/rook/rook/issues/2021
+ hostNetwork: true
+ volumes:
+ - name: dev
+ hostPath:
+ path: /dev
+ - name: sysbus
+ hostPath:
+ path: /sys/bus
+ - name: libmodules
+ hostPath:
+ path: /lib/modules
+ - name: mon-endpoint-volume
+ configMap:
+ name: rook-ceph-mon-endpoints
+ items:
+ - key: data
+ path: mon-endpoints
diff --git a/playbooks/k8s/storage-class.yaml b/playbooks/k8s/storage-class.yaml
new file mode 100644
index 0000000000..a6e0b4d60f
--- /dev/null
+++ b/playbooks/k8s/storage-class.yaml
@@ -0,0 +1,11 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+ name: cinder
+ labels:
+ kubernetes.io/cluster-service: "true"
+ addonmanager.kubernetes.io/mode: EnsureExists
+provisioner: kubernetes.io/cinder
+parameters:
+ type: rbd
+ availability: nova
diff --git a/playbooks/run-k8s-on-openstack.yaml b/playbooks/run-k8s-on-openstack.yaml
index eee4ae2d07..4efcfacd56 100644
--- a/playbooks/run-k8s-on-openstack.yaml
+++ b/playbooks/run-k8s-on-openstack.yaml
@@ -15,3 +15,8 @@
command: ./run_k8s_ansible.sh
args:
chdir: /opt/system-config
+
+ - name: Install cinder storage class
+ k8s:
+ state: present
+ definition: "{{ lookup('file', 'k8s/storage-class.yaml') | from_yaml }}"