Implement cinder

Depends-On: https://review.opendev.org/747030

Change-Id: If8e45498efa9c7acfb8962e1fcd40accacf13a7a

Makefile

@@ -2,6 +2,7 @@ images:
 	docker build images/horizon -t vexxhost/horizon:latest
 	docker build images/keystone -t vexxhost/keystone:latest
 	docker build images/ceilometer --target ceilometer-agent-notification -t vexxhost/ceilometer-agent-notification:latest
+	docker build images/cinder --target cinder-api -t vexxhost/cinder-api:latest
 	docker build images/heat --target heat-api -t vexxhost/heat-api:latest
 	docker build images/heat --target heat-api-cfn -t vexxhost/heat-api-cfn:latest
 	docker build images/heat --target heat-engine -t vexxhost/heat-engine:latest

chart/test-values.yaml

@@ -4,6 +4,7 @@ configMap:
   barbican: {}
   ceilometer:
     dbUri: "sqlite:///:memory:"
+  cinder: {}
   glance: {}
   placement: {}
   neutron: {}

config/samples/operator-config.yaml

@@ -5,6 +5,7 @@ metadata:
 data:
   operator-config.yaml: |
     barbican: {}
+    cinder: {}
     ceilometer:
       dbUri: "sqlite:///:memory:"
     horizon:

devstack/lib/cinder

@@ -0,0 +1,136 @@
+#!/bin/bash
+#
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+CINDER_STATE_PATH=/var/lib/cinder
+
+# configure_cinder() - Set config files, create data dirs, etc
+function configure_cinder {
+    sudo install -d -o $STACK_USER -m 755 $CINDER_CONF_DIR
+
+    rm -f $CINDER_CONF
+
+    configure_rootwrap cinder
+    sudo sed -e "s:^filters_path=.*$:filters_path=/usr/local/etc/cinder/rootwrap.d:" -i $CINDER_CONF_DIR/rootwrap.conf
+    cp -p "$CINDER_DIR/etc/cinder/resource_filters.json" "$CINDER_CONF_DIR/resource_filters.json"
+    cp $CINDER_DIR/etc/cinder/api-paste.ini $CINDER_API_PASTE_INI
+
+    kubernetes_ensure_resource secret/cinder-application-credential
+    CINDER_APPLICATION_CREDENTIAL_SECRET=$(get_data_from_secret cinder-application-credential openstack secret)
+    CINDER_APPLICATION_CREDENTIAL_ID=$(get_data_from_secret cinder-application-credential openstack id)
+    iniset $CINDER_CONF keystone_authtoken auth_url $KEYSTONE_AUTH_URI_V3
+    iniset $CINDER_CONF keystone_authtoken auth_type v3applicationcredential
+    iniset $CINDER_CONF keystone_authtoken application_credential_id $CINDER_APPLICATION_CREDENTIAL_ID
+    iniset $CINDER_CONF keystone_authtoken application_credential_secret $CINDER_APPLICATION_CREDENTIAL_SECRET
+
+    iniset $CINDER_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
+
+    # NOTE(Alex): operator stuff
+    kubernetes_ensure_resource secret/cinder-mysql
+    CINDER_DATABASE_USER=$(get_data_from_secret cinder-mysql openstack USER)
+    CINDER_DATABASE_PASSWORD=$(get_data_from_secret cinder-mysql openstack PASSWORD)
+    CINDER_DATABASE_NAME=$(get_data_from_secret cinder-mysql openstack DATABASE)
+    iniset $CINDER_CONF database connection "mysql+pymysql://$CINDER_DATABASE_USER:$CINDER_DATABASE_PASSWORD@cinder-mysql-master/$CINDER_DATABASE_NAME?charset=utf8"
+
+    iniset $CINDER_CONF DEFAULT api_paste_config $CINDER_API_PASTE_INI
+    iniset $CINDER_CONF DEFAULT rootwrap_config "$CINDER_CONF_DIR/rootwrap.conf"
+    iniset $CINDER_CONF DEFAULT osapi_volume_extension cinder.api.contrib.standard_extensions
+    iniset $CINDER_CONF DEFAULT osapi_volume_listen $CINDER_SERVICE_LISTEN_ADDRESS
+    iniset $CINDER_CONF DEFAULT state_path $CINDER_STATE_PATH
+    iniset $CINDER_CONF DEFAULT my_ip "$HOST_IP"
+    iniset $CINDER_CONF key_manager backend cinder.keymgr.conf_key_mgr.ConfKeyManager
+    iniset $CINDER_CONF key_manager fixed_key $(openssl rand -hex 16)
+
+    configure_cinder_backend_ceph ceph
+    iniset $CINDER_CONF ceph volume_clear $CINDER_VOLUME_CLEAR
+    iniset $CINDER_CONF DEFAULT enabled_backends ceph
+    iniset $CINDER_CONF DEFAULT default_volume_type ceph
+    configure_cinder_image_volume_cache
+
+    iniset $CINDER_CONF DEFAULT backup_swift_url "$SWIFT_SERVICE_PROTOCOL://$SERVICE_HOST:$SWIFT_DEFAULT_BIND_PORT/v1/AUTH_"
+    iniset $CINDER_CONF oslo_messaging_notifications driver "messagingv2"
+
+    # Get rabbitmq password
+    kubernetes_ensure_resource secret/cinder-rabbitmq
+    CINDER_RABBITMQ_PASSWORD=$(get_data_from_secret cinder-rabbitmq openstack password)
+    CINDER_RABBITMQ_USERNAME=$(get_data_from_secret cinder-rabbitmq openstack username)
+    iniset_k8s_rpc_backend cinder $CINDER_CONF DEFAULT "rabbit://$CINDER_RABBITMQ_USERNAME:$CINDER_RABBITMQ_PASSWORD@rabbitmq-cinder:5672/"
+
+    if [[ -r $CINDER_PLUGINS/$CINDER_DRIVER ]]; then
+        configure_cinder_driver
+    fi
+
+    iniset $CINDER_CONF DEFAULT osapi_volume_workers "$API_WORKERS"
+
+    iniset $CINDER_CONF DEFAULT glance_api_servers "http://glance"
+
+    # Set nova credentials (used for os-assisted-snapshots)
+
+    iniset $CINDER_CONF nova auth_type password
+    iniset $CINDER_CONF nova auth_url $KEYSTONE_SERVICE_URI
+    iniset $CINDER_CONF nova username nova
+    iniset $CINDER_CONF nova password $SERVICE_PASSWORD
+    iniset $CINDER_CONF nova user_domain_name "$SERVICE_DOMAIN_NAME"
+    iniset $CINDER_CONF nova project_name $SERVICE_PROJECT_NAME
+    iniset $CINDER_CONF nova project_domain_name "$SERVICE_DOMAIN_NAME"
+
+    iniset $CINDER_CONF nova region_name "$REGION_NAME"
+    iniset $CINDER_CONF DEFAULT graceful_shutdown_timeout "$SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT"
+
+}
+export -f configure_cinder
+
+
+# init_cinder() - Initialize database and volume group
+function init_cinder {
+    echo noop
+}
+
+# install_cinder() - Collect source and prepare
+function install_cinder {
+    echo noop
+}
+
+# install_cinderclient() - Collect source and prepare
+# NOTE(Alex): I am not sure this function is not overrided if the content is empty.
+#             So I remove this function for now in this override lib file.
+# function install_cinderclient {
+#     echo noop
+# }
+
+function cleanup_cinder {
+    echo noop
+}
+
+# start_cinder() - Start running processes
+function start_cinder {
+
+	kubernetes_rollout_restart daemonset/cinder-api
+    kubernetes_rollout_restart daemonset/cinder-scheduler
+    kubernetes_rollout_restart daemonset/cinder-volume
+
+	kubernetes_rollout_status daemonset/cinder-api
+	kubernetes_rollout_status daemonset/cinder-scheduler
+	kubernetes_rollout_status daemonset/cinder-volume
+
+	proxy_pass_to_kubernetes /volume cinder cinder-wsgi
+
+    sleep 10
+}
+
+# stop_cinder() - Stop running processes
+function stop_cinder {
+    echo noop
+}

devstack/lib/common

@@ -31,7 +31,7 @@ function kubernetes_rollout_status {
 		kubectl get $resource && break || sleep 1;
 	done
 
-	kubectl rollout status --timeout=60s $resource
+	kubectl rollout status --timeout=300s $resource
 }
 
 function kubernetes_rollout_restart {
@@ -46,8 +46,9 @@ function kubernetes_rollout_restart {
 
 function kubernetes_ensure_resource {
 	local resource="$1"
-	for i in {1..60}; do
+	kubectl logs deployment/openstack-operator -n default
-		kubectl get $resource && break || sleep 3;
+	for i in {1..120}; do
+		kubectl get $resource && break || sleep 5;
 	done
 }
 

devstack/plugin.sh

@@ -41,11 +41,18 @@ elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
     --from-file=/etc/glance/glance-api.conf \
     --from-file=/etc/glance/glance-api-paste.ini
 
+    kubectl create secret generic cinder-config -n openstack \
+    --from-file=/etc/cinder/cinder.conf \
+    --from-file=/etc/cinder/api-paste.ini \
+    --from-file=/etc/cinder/rootwrap.conf \
+    --from-file=/etc/cinder/resource_filters.json
+
 	# NOTE(Alex): Permissions here are bad but it's temporary so we don't care as much.
 	sudo chmod -Rv 777 /etc/ceph
     kubectl create secret generic ceph-config -n openstack \
     --from-file=/etc/ceph/ceph.conf \
-	--from-file=/etc/ceph/ceph.client.glance.keyring
+	--from-file=/etc/ceph/ceph.client.glance.keyring \
+	--from-file=/etc/ceph/ceph.client.cinder.keyring
 
 elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
 	:

devstack/settings

@@ -18,6 +18,7 @@ define_plugin openstack-operator
 
 source $DEST/openstack-operator/devstack/lib/common
 source $DEST/openstack-operator/devstack/lib/barbican
+source $DEST/openstack-operator/devstack/lib/cinder
 source $DEST/openstack-operator/devstack/lib/glance
 source $DEST/openstack-operator/devstack/lib/horizon
 source $DEST/openstack-operator/devstack/lib/keystone

images/cinder/Dockerfile

@@ -0,0 +1,33 @@
+# Copyright (c) 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM vexxhost/python-builder as builder
+FROM vexxhost/python-base AS cinder-base
+RUN mkdir -p /var/lib/cinder && \
+    chmod 777 -R /var/lib/cinder
+
+FROM cinder-base AS cinder-api
+COPY cinder-wsgi /usr/local/bin/cinder-wsgi
+EXPOSE 8776
+ENV UWSGI_HTTP_SOCKET=:8776 UWSGI_WSGI_FILE=/usr/local/bin/cinder-wsgi
+CMD ["/usr/local/bin/uwsgi", "--ini", "/etc/uwsgi/uwsgi.ini"]
+
+FROM cinder-base AS cinder-volume
+COPY cinder-volume /usr/local/bin/cinder-volume
+CMD ["/usr/local/bin/cinder-volume"]
+
+FROM cinder-base AS cinder-scheduler
+COPY cinder-scheduler /usr/local/bin/cinder-scheduler
+CMD ["/usr/local/bin/cinder-scheduler"]

images/cinder/bindep.txt

@@ -0,0 +1,9 @@
+gcc [compile]
+libc-dev [compile]
+librados-dev [compile]
+librbd-dev [compile]
+librados2
+librbd1
+ceph-common
+qemu-utils
+sudo

images/cinder/build-requirements.txt

@@ -0,0 +1 @@
+Cython

images/cinder/cinder-scheduler

@@ -0,0 +1,30 @@
+#!/usr/local/bin/python
+# Copyright (c) 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pkg_resources
+import re
+import sys
+
+import sentry_sdk
+
+from cinder.cmd.scheduler import main
+
+VERSION = pkg_resources.get_distribution("cinder").version
+
+sentry_sdk.init(release="cinder@%s" % VERSION)
+
+sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+sys.exit(main())

images/cinder/cinder-volume

@@ -0,0 +1,30 @@
+#!/usr/local/bin/python
+# Copyright (c) 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pkg_resources
+import re
+import sys
+
+import sentry_sdk
+
+from cinder.cmd.volume import main
+
+VERSION = pkg_resources.get_distribution("cinder").version
+
+sentry_sdk.init(release="cinder@%s" % VERSION)
+
+sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+sys.exit(main())

images/cinder/cinder-wsgi

@@ -0,0 +1,29 @@
+#!/usr/local/bin/python
+# Copyright (c) 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pkg_resources
+
+import sentry_sdk
+
+from cinder.wsgi.wsgi import initialize_application
+from sentry_sdk.integrations import wsgi
+
+VERSION = pkg_resources.get_distribution("cinder").version
+
+sentry_sdk.init(release="cinder@%s" % VERSION)
+
+application = initialize_application()
+application = wsgi.SentryWsgiMiddleware(application)

images/cinder/constraints.txt

@@ -0,0 +1 @@
+--constraint https://releases.openstack.org/constraints/upper/ussuri

images/cinder/requirements.txt

@@ -0,0 +1,8 @@
+uWSGI
+boto3
+PyMySQL
+python-memcached
+sentry-sdk
+git+https://opendev.org/openstack/cinder@stable/ussuri
+https://github.com/ceph/ceph/archive/v15.2.4.tar.gz#egg=rados&subdirectory=src/pybind/rados
+https://github.com/ceph/ceph/archive/v15.2.4.tar.gz#egg=rbd&subdirectory=src/pybind/rbd

images/cinder/setup-repos.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# Copyright (c) 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+
+apt-get install -y gnupg2
+
+cat <<EOF | apt-key add -
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQINBFX4hgkBEADLqn6O+UFp+ZuwccNldwvh5PzEwKUPlXKPLjQfXlQRig1flpCH
+E0HJ5wgGlCtYd3Ol9f9+qU24kDNzfbs5bud58BeE7zFaZ4s0JMOMuVm7p8JhsvkU
+C/Lo/7NFh25e4kgJpjvnwua7c2YrA44ggRb1QT19ueOZLK5wCQ1mR+0GdrcHRCLr
+7Sdw1d7aLxMT+5nvqfzsmbDullsWOD6RnMdcqhOxZZvpay8OeuK+yb8FVQ4sOIzB
+FiNi5cNOFFHg+8dZQoDrK3BpwNxYdGHsYIwU9u6DWWqXybBnB9jd2pve9PlzQUbO
+eHEa4Z+jPqxY829f4ldaql7ig8e6BaInTfs2wPnHJ+606g2UH86QUmrVAjVzlLCm
+nqoGymoAPGA4ObHu9X3kO8viMBId9FzooVqR8a9En7ZE0Dm9O7puzXR7A1f5sHoz
+JdYHnr32I+B8iOixhDUtxIY4GA8biGATNaPd8XR2Ca1hPuZRVuIiGG9HDqUEtXhV
+fY5qjTjaThIVKtYgEkWMT+Wet3DPPiWT3ftNOE907e6EWEBCHgsEuuZnAbku1GgD
+LBH4/a/yo9bNvGZKRaTUM/1TXhM5XgVKjd07B4cChgKypAVHvef3HKfCG2U/DkyA
+LjteHt/V807MtSlQyYaXUTGtDCrQPSlMK5TjmqUnDwy6Qdq8dtWN3DtBWQARAQAB
+tCpDZXBoLmNvbSAocmVsZWFzZSBrZXkpIDxzZWN1cml0eUBjZXBoLmNvbT6JAjgE
+EwECACIFAlX4hgkCGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEOhKwsBG
+DzmUXdIQAI8YPcZMBWdv489q8CzxlfRIRZ3Gv/G/8CH+EOExcmkVZ89mVHngCdAP
+DOYCl8twWXC1lwJuLDBtkUOHXNuR5+Jcl5zFOUyldq1Hv8u03vjnGT7lLJkJoqpG
+l9QD8nBqRvBU7EM+CU7kP8+09b+088pULil+8x46PwgXkvOQwfVKSOr740Q4J4nm
+/nUOyTNtToYntmt2fAVWDTIuyPpAqA6jcqSOC7Xoz9cYxkVWnYMLBUySXmSS0uxl
+3p+wK0lMG0my/gb+alke5PAQjcE5dtXYzCn+8Lj0uSfCk8Gy0ZOK2oiUjaCGYN6D
+u72qDRFBnR3jaoFqi03bGBIMnglGuAPyBZiI7LJgzuT9xumjKTJW3kN4YJxMNYu1
+FzmIyFZpyvZ7930vB2UpCOiIaRdZiX4Z6ZN2frD3a/vBxBNqiNh/BO+Dex+PDfI4
+TqwF8zlcjt4XZ2teQ8nNMR/D8oiYTUW8hwR4laEmDy7ASxe0p5aijmUApWq5UTsF
++s/QbwugccU0iR5orksM5u9MZH4J/mFGKzOltfGXNLYI6D5Mtwrnyi0BsF5eY0u6
+vkdivtdqrq2DXY+ftuqLOQ7b+t1RctbcMHGPptlxFuN9ufP5TiTWSpfqDwmHCLsT
+k2vFiMwcHdLpQ1IH8ORVRgPPsiBnBOJ/kIiXG2SxPUTjjEGOVgeA
+=/Tod
+-----END PGP PUBLIC KEY BLOCK-----
+EOF
+
+cat <<EOF | tee /etc/apt/sources.list.d/ceph.list
+deb https://download.ceph.com/debian-octopus/ buster main
+EOF

openstack_operator/cinder.py

@@ -0,0 +1,86 @@
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""cinder Operator
+
+This module maintains the operator for Cinder.
+"""
+
+from openstack_operator import database
+from openstack_operator import identity
+from openstack_operator import utils
+
+MEMCACHED = True
+
+
+def create_or_resume(name, spec, **_):
+    """Create and re-sync a cinder instance
+
+    This function is called when a new resource is created but also when we
+    start the service up for the first time.
+    """
+
+    # deploy mysql for cinder
+    if "mysql" not in spec:
+        database.ensure_mysql_cluster("cinder", {})
+    else:
+        database.ensure_mysql_cluster("cinder", spec["mysql"])
+
+    # deploy rabbitmq
+    if not utils.ensure_secret("openstack", "cinder-rabbitmq"):
+        utils.create_or_update('cinder/secret-rabbitmq.yml.j2',
+                               password=utils.generate_password())
+    utils.create_or_update('cinder/rabbitmq.yml.j2', spec=spec)
+
+    # deploy cinder
+    config_hash = utils.generate_hash(spec)
+
+    for component in ("api", "scheduler", "volume"):
+        utils.create_or_update('cinder/daemonset.yml.j2',
+                               name=name, spec=spec,
+                               component=component,
+                               config_hash=config_hash)
+
+    utils.create_or_update('cinder/service.yml.j2', name=name)
+
+    url = None
+    if "ingress" in spec:
+        utils.create_or_update('cinder/ingress.yml.j2',
+                               name=name, spec=spec)
+
+        url = spec["ingress"]["host"]
+
+    # Create application credential
+    identity.ensure_application_credential(name="cinder")
+
+    identity.ensure_service(name="cinder", service_type="block-storage",
+                            url=url, desc="Cinder Volume Service",
+                            path="/v3/$(project_id)s")
+    identity.ensure_service(name="cinderv2", service_type="volumev2",
+                            url=url, desc="Cinder Volume Service V2",
+                            path="/v2/$(project_id)s", internal="cinder")
+    identity.ensure_service(name="cinderv3", service_type="volumev3",
+                            url=url, desc="Cinder Volume Service V3",
+                            path="/v3/$(project_id)s", internal="cinder")
+
+
+def update(name, spec, **_):
+    """Update a cinder
+
+    This function updates the deployment for cinder if there are any
+    changes that happen within it.
+    """
+    if "ingress" in spec:
+        utils.create_or_update('cinder/ingress.yml.j2',
+                               name=name, spec=spec)

openstack_operator/identity.py

@@ -17,16 +17,20 @@
 This module contains a few common functions for identity management
 """
 
+# pylint: disable=R0913
+
 from openstack_operator import utils
 
 
-def ensure_service(name, service_type, desc, url=None, path=""):
+def ensure_service(name, service_type, desc, url=None,
+                   internal=None, path=""):
     """Create or update service and endpoints
 
     name: service name
     service_type: service type
     desc: service descriptioin
     url: hostname of public endpoint
+    internal: hostname of internal endpoint
     path: sub path of endpoint
     """
 
@@ -35,8 +39,10 @@ def ensure_service(name, service_type, desc, url=None, path=""):
                            type=service_type, description=desc)
 
     # Create or resume endpoints
+    if internal is None:
+        internal = name
     internal_url = public_url = \
-        "http://" + name + ".openstack.svc.cluster.local" + path
+        "http://" + internal + ".openstack.svc.cluster.local" + path
 
     if url is not None:
         public_url = "https://" + url + path

openstack_operator/operator.py

@@ -29,6 +29,7 @@ from sentry_sdk.integrations import aiohttp
 from openstack_operator import barbican
 from openstack_operator import ceilometer
 from openstack_operator import chronyd
+from openstack_operator import cinder
 from openstack_operator import glance
 from openstack_operator import heat
 from openstack_operator import horizon
@@ -108,6 +109,9 @@ def deploy(name, namespace, new, **_):
     if "glance" in config:
         spec = set_service_config(config, "glance")
         glance.create_or_resume("glance", spec)
+    if "cinder" in config:
+        spec = set_service_config(config, "cinder")
+        cinder.create_or_resume("cinder", spec)
     if "magnum" in config:
         spec = set_service_config(config, "magnum")
         magnum.create_or_resume("magnum", spec)

openstack_operator/templates/cinder/daemonset.yml.j2

@@ -0,0 +1,120 @@
+---
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cinder-{{ component }}
+  namespace: openstack
+  labels:
+    {{ labels("cinder", component=component) | indent(4) }}
+spec:
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      {{ labels("cinder", component=component) | indent(6) }}
+  template:
+    metadata:
+      labels:
+        {{ labels("cinder", component=component) | indent(8) }}
+      annotations:
+        checksum/config: "{{ config_hash }}"
+    spec:
+      automountServiceAccountToken: false
+      {% if 'scheduler' in component %}
+      initContainers:
+      - name: db-sync
+        image: vexxhost/cinder-{{ component }}:latest
+        imagePullPolicy: Always
+        command:
+        - cinder-manage
+        - db
+        - sync
+        volumeMounts:
+        - mountPath: /etc/cinder
+          name: cinder-config
+      {% endif %}
+      containers:
+      - name: cinder-{{ component }}
+        image: vexxhost/cinder-{{ component }}:latest
+        imagePullPolicy: Always
+        env:
+        {% if 'api' not in component %}
+        - name: OS_DEFAULT__HOST
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        {% endif %}
+        {% if 'sentryDSN' in spec %}
+        - name: SENTRY_DSN
+          value: {{ spec.sentryDSN }}
+        {% endif %}
+        {% for v in env %}
+        - name: "{{ v.name }}"
+          value: "{{ v.value }}"
+        {% endfor %}
+        {% if 'api' in component %}
+        ports:
+        - name: cinder
+          protocol: TCP
+          containerPort: 8776
+        livenessProbe:
+          tcpSocket:
+            port: cinder
+        readinessProbe:
+          tcpSocket:
+            port: cinder
+        {% endif %}
+        {% if 'volume' not in component %}
+        securityContext:
+          runAsUser: 1001
+        {% endif %}
+        volumeMounts:
+        {% if 'volume' in component %}
+        - name: ceph-config
+          mountPath: /etc/ceph
+        {% endif %}
+        - name: cinder-config
+          mountPath: /etc/cinder
+        - name: uwsgi-config
+          mountPath: /etc/uwsgi
+      volumes:
+      {% if 'volume' in component %}
+      - name: ceph-config
+        secret:
+          secretName: ceph-config
+      {% endif %}
+      - name: cinder-config
+        secret:
+          secretName: cinder-config
+      - name: uwsgi-config
+        configMap:
+          defaultMode: 420
+          name: uwsgi-default
+{% if 'nodeSelector' in spec %}
+      nodeSelector:
+        {{ spec.nodeSelector | to_yaml | indent(8) }}
+{% endif %}
+{% if 'tolerations' in spec %}
+      tolerations:
+        {{ spec.tolerations | to_yaml | indent(8) }}
+{% endif %}
+{% if 'hostAliases' in spec %}
+      hostAliases:
+        {{ spec.hostAliases | to_yaml | indent(8) }}
+{% endif %}

openstack_operator/templates/cinder/ingress.yml.j2

@@ -0,0 +1,55 @@
+---
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: networking.k8s.io/v1beta1
+kind: Ingress
+metadata:
+  name: cinder
+  namespace: openstack
+  annotations:
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+    certmanager.k8s.io/cluster-issuer: "letsencrypt-prod"
+spec:
+{% if spec.ingress.host is defined %}
+  rules:
+  - host: {{ spec.ingress.host }}
+    http:
+      paths:
+      - path: /
+        backend:
+          serviceName: cinder
+          servicePort: 80
+  tls:
+  - hosts:
+    - {{ spec.ingress.host }}
+    secretName: cinder-tls
+{% else %}
+  rules:
+  {% for v in spec.ingress %}
+  - host: {{ v.host }}
+    http:
+      paths:
+      - path: /
+        backend:
+          serviceName: cinder
+          servicePort: 80
+  {% endfor %}
+  tls:
+  - hosts:
+    {% for v in spec.ingress %}
+    - {{ v.host }}
+    {% endfor %}
+    secretName: cinder-tls
+{% endif %}

openstack_operator/templates/cinder/rabbitmq.yml.j2

@@ -0,0 +1,27 @@
+---
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: infrastructure.vexxhost.cloud/v1alpha1
+kind: Rabbitmq
+metadata:
+  name: cinder
+  namespace: openstack
+spec:
+  authSecret: cinder-rabbitmq
+  nodeSelector:
+    node-role.kubernetes.io/master: ""
+  tolerations:
+  - key: node-role.kubernetes.io/master
+    effect: NoSchedule

openstack_operator/templates/cinder/secret-rabbitmq.yml.j2

@@ -0,0 +1,9 @@
+
+apiVersion: v1
+metadata:
+  name: cinder-rabbitmq
+  namespace: openstack
+stringData:
+  username: cinder
+  password: {{ password }}
+kind: Secret

openstack_operator/templates/cinder/service.yml.j2

@@ -0,0 +1,28 @@
+---
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: cinder
+  namespace: openstack
+spec:
+  ports:
+  - name: cinder
+    port: 80
+    protocol: TCP
+    targetPort: cinder
+  selector:
+    {{ labels("cinder", component="api") | indent(4) }}

zuul.d/cinder-jobs.yaml

@@ -0,0 +1,51 @@
+- job:
+    name: openstack-operator:images:build:cinder
+    parent: vexxhost-build-docker-image
+    provides: openstack-operator:image:cinder
+    nodeset: &id001
+      nodes:
+      - name: ubuntu-bionic
+        label: ubuntu-bionic-vexxhost
+    vars: &id002
+      docker_images:
+      - context: images/cinder
+        repository: vexxhost/cinder-base
+        target: cinder-base
+      - context: images/cinder
+        repository: vexxhost/cinder-api
+        target: cinder-api
+      - context: images/cinder
+        repository: vexxhost/cinder-volume
+        target: cinder-volume
+      - context: images/cinder
+        repository: vexxhost/cinder-scheduler
+        target: cinder-scheduler
+    dependencies:
+    - openstack-operator:images:build:openstack-operator
+    files: &id003
+    - ^images/cinder/.*
+- job:
+    name: openstack-operator:images:upload:cinder
+    parent: vexxhost-upload-docker-image
+    provides: openstack-operator:image:cinder
+    nodeset: *id001
+    vars: *id002
+    dependencies:
+    - openstack-operator:images:upload:openstack-operator
+    files: *id003
+- job:
+    name: openstack-operator:images:promote:cinder
+    parent: vexxhost-promote-docker-image
+    nodeset: *id001
+    vars: *id002
+    files: *id003
+- project:
+    check:
+      jobs:
+      - openstack-operator:images:build:cinder
+    gate:
+      jobs:
+      - openstack-operator:images:upload:cinder
+    promote:
+      jobs:
+      - openstack-operator:images:promote:cinder

zuul.d/functional-jobs.yaml

@@ -27,6 +27,7 @@
             identity-feature-enabled:
               application_credentials: true
       devstack_services:
+        c-bak: false
         etcd3: false
         horizon: true
         rabbit: false
@@ -61,65 +62,69 @@
       jobs:
       - openstack-operator:functional:
           dependencies:
-          - name: openstack-operator:images:build:heat
+          - name: openstack-operator:images:build:mcrouter-exporter
             soft: true
-          - name: openstack-operator:images:build:barbican
+          - name: openstack-operator:images:build:horizon
             soft: true
-          - name: openstack-operator:images:build:ceilometer
+          - name: openstack-operator:images:build:placement
+            soft: true
+          - name: openstack-operator:images:build:magnum
+            soft: true
+          - name: openstack-operator:images:build:glance
             soft: true
           - name: openstack-operator:images:build:neutron
             soft: true
           - name: openstack-operator:images:build:rabbitmq
             soft: true
-          - name: openstack-operator:images:build:keystone
+          - name: openstack-operator:images:build:ceilometer
-            soft: true
-          - name: openstack-operator:images:build:horizon
-            soft: true
-          - name: openstack-operator:images:build:memcached
-            soft: true
-          - name: openstack-operator:images:build:mcrouter
-            soft: true
-          - openstack-operator:images:build:openstack-operator
-          - name: openstack-operator:images:build:glance
-            soft: true
-          - name: openstack-operator:images:build:mcrouter-exporter
             soft: true
           - name: openstack-operator:images:build:memcached-exporter
             soft: true
-          - name: openstack-operator:images:build:magnum
+          - name: openstack-operator:images:build:memcached
             soft: true
-          - name: openstack-operator:images:build:placement
+          - name: openstack-operator:images:build:keystone
+            soft: true
+          - name: openstack-operator:images:build:mcrouter
+            soft: true
+          - name: openstack-operator:images:build:cinder
+            soft: true
+          - openstack-operator:images:build:openstack-operator
+          - name: openstack-operator:images:build:barbican
+            soft: true
+          - name: openstack-operator:images:build:heat
             soft: true
     gate:
       jobs:
       - openstack-operator:functional:
           dependencies:
-          - name: openstack-operator:images:upload:heat
+          - name: openstack-operator:images:upload:mcrouter-exporter
             soft: true
-          - name: openstack-operator:images:upload:barbican
+          - name: openstack-operator:images:upload:horizon
             soft: true
-          - name: openstack-operator:images:upload:ceilometer
+          - name: openstack-operator:images:upload:placement
+            soft: true
+          - name: openstack-operator:images:upload:magnum
+            soft: true
+          - name: openstack-operator:images:upload:glance
             soft: true
           - name: openstack-operator:images:upload:neutron
             soft: true
           - name: openstack-operator:images:upload:rabbitmq
             soft: true
-          - name: openstack-operator:images:upload:keystone
+          - name: openstack-operator:images:upload:ceilometer
-            soft: true
-          - name: openstack-operator:images:upload:horizon
-            soft: true
-          - name: openstack-operator:images:upload:memcached
-            soft: true
-          - name: openstack-operator:images:upload:mcrouter
-            soft: true
-          - openstack-operator:images:upload:openstack-operator
-          - name: openstack-operator:images:upload:glance
-            soft: true
-          - name: openstack-operator:images:upload:mcrouter-exporter
             soft: true
           - name: openstack-operator:images:upload:memcached-exporter
             soft: true
-          - name: openstack-operator:images:upload:magnum
+          - name: openstack-operator:images:upload:memcached
             soft: true
-          - name: openstack-operator:images:upload:placement
+          - name: openstack-operator:images:upload:keystone
+            soft: true
+          - name: openstack-operator:images:upload:mcrouter
+            soft: true
+          - name: openstack-operator:images:upload:cinder
+            soft: true
+          - openstack-operator:images:upload:openstack-operator
+          - name: openstack-operator:images:upload:barbican
+            soft: true
+          - name: openstack-operator:images:upload:heat
             soft: true