From 56b0233ffeec2e9ff3b4f10f7e1b7116451f6eb1 Mon Sep 17 00:00:00 2001
From: Adam Coldrick <adam.coldrick@codethink.co.uk>
Date: Wed, 20 Mar 2019 16:11:33 +0000
Subject: [PATCH] Add docker-compose based dev setup instructions
This adds a docker-compose.yml file which runs the various services
required for a working development instance of StoryBoard (and also
Swift, which technically isn't needed yet). It also provides a config
file which is pre-configured to work out of the box with the services
in the docker-compose.yml file.
It also updates the development installation instructions to recommend
using docker-compose to run the services, rather than installing them
on a machine and configuring everything manually.
Change-Id: I405e2e46f6ab7d8ef2c1ddd43b3e2e8cb8e3a808
---
doc/source/install/development.rst | 175 +++++++++++++++++++++++++
docker/.env | 1 +
docker/docker-compose.yml | 26 ++++
docker/storyboard.conf | 204 +++++++++++++++++++++++++++++
4 files changed, 406 insertions(+)
create mode 100644 docker/.env
create mode 100644 docker/docker-compose.yml
create mode 100644 docker/storyboard.conf
diff --git a/doc/source/install/development.rst b/doc/source/install/development.rst
index 8f3bc501..51c0a86b 100644
--- a/doc/source/install/development.rst
+++ b/doc/source/install/development.rst
@@ -15,6 +15,181 @@ This install guide will cover the API and the most widely-used
StoryBoard webclient, and assumes being run on Ubuntu 16.04 or
newer. The instructions are mostly portable to other distributions.
+The recommended way to set up your machine for developing StoryBoard
+is to use the docker-compose.yml file provided. However, we also
+provide instructions for a manual setup if preferred.
+
+
+Using Docker
+============
+
+This approach uses Docker to run the services required by StoryBoard,
+such as MySQL and RabbitMQ. The StoryBoard API and webclient are run
+on the host machine directly, to reduce cycle time when developing.
+They use ``tox`` to run using virtualenvs to minimise the amount of
+manual installation required.
+
+Upon completion of these steps, you should have a usable StoryBoard
+API running at ``http://localhost:8080/`` and a usable StoryBoard
+webclient served at ``http://localhost:9000/``.
+
+
+1. Install docker
+-----------------
+
+Follow the `docker installation instructions
+<https://docs.docker.com/install/>`_ for your platform.
+
+.. note:: On Linux, be sure to add your user to the docker group to
+ avoid needing sudo::
+
+ sudo usermod -aG docker your-user
+
+ You'll need to log out and in again for this to take effect.
+
+
+2. Install docker-compose
+-------------------------
+
+Either install using pip::
+
+ pip3 install --user docker-compose
+
+or follow `the instructions
+<https://docs.docker.com/compose/install/>`_ for your platform.
+
+
+3. Get the code
+---------------
+
+The code is stored using git, so you'll need to have git installed::
+
+ sudo apt install git
+
+The code for the API and webclient can then be cloned::
+
+ git clone https://git.openstack.org/openstack-infra/storyboard
+ git clone https://git.openstack.org/openstack-infra/storyboard-webclient
+ cd storyboard
+
+
+4. Run containers
+-----------------
+
+Currently the docker-compose.yml file sets up 3 containers to
+provide the following services
+
+- MySQL
+- Swift
+- RabbitMQ
+
+The containers can be started by doing the following, starting in the
+root of the ``storyboard`` repository::
+
+ cd docker
+ docker-compose up
+
+.. note:: You can make the docker-compose process run in the background
+ by instead doing::
+
+ cd docker
+ docker-compose up -d
+
+
+5. Install dependencies
+-----------------------
+
+Some dependencies are needed to run the API and build the webclient. On
+Ubuntu, you can install these with::
+
+ sudo apt install build-essential python3-dev
+ pip3 install --user tox
+
+
+6. Migrate the database
+-----------------------
+
+At this point you could run StoryBoard, but its useless with an empty
+database. The migrations are run using the ``storyboard-db-manage``
+script, which you can run using tox in the root of the ``storyboard``
+repository::
+
+ tox -e venv -- storyboard-db-manage --config-file ./docker/storyboard.conf upgrade head
+
+This command runs all the database migrations in order. Under the hood
+it uses `alembic <https://alembic.sqlalchemy.org/en/latest/>`_, and
+has a similar CLI.
+
+
+7. Run the API
+--------------
+
+The API is run using the ``storyboard-api`` command. Again this can
+be run using tox in the root of the ``storyboard`` repository::
+
+ tox -e venv -- storyboard-api --config-file ./docker/storyboard.conf
+
+The ``docker/storyboard.conf`` configuration file is contains config
+which is already set up to use the containers created earlier, so
+there is no need for manual configuration.
+
+The output of this command should finish with something like::
+
+ 2019-03-20 11:25:44.862 22047 INFO storyboard.api.app [-] Starting server in PID 22047
+ 2019-03-20 11:25:44.863 22047 INFO storyboard.api.app [-] Configuration:
+ 2019-03-20 11:25:44.863 22047 INFO storyboard.api.app [-] serving on 0.0.0.0:8080, view at http://127.0.0.1:8080
+
+At that point, the API is running successfully. You can stop it using
+Ctrl+C or by closing your terminal.
+
+
+8. Serve the webclient
+----------------------
+
+The storyboard-webclient repository provides a tox target which builds
+the webclient and serves it using a development server. You can run it
+using tox in the root of the ``storyboard-webclient`` repository::
+
+ tox -e grunt_no_api -- serve
+
+This will take a little while to run as it obtains the required dependencies
+using ``npm``, and builds node-sass.
+
+The output of this command should finish with something like::
+
+ Running "connect:livereload" (connect) task
+ Started connect web server on http://localhost:9000
+
+ Running "watch" task
+ Waiting...
+
+At that point the webclient is being served successfully. You can stop it
+using Ctrl+C or by closing the terminal. Any changes to existing files in
+the codebase will cause it to automatically rebuild the webclient and
+refresh the page in your browser, to help streamline the development
+workflow.
+
+You can view it in a browser at ``http://localhost:9000/``. You should also
+be able to log in here. The provided configuration file uses Ubuntu One as
+the OpenID provider, so you'll need an Ubuntu One account to do so.
+
+
+9. Enable notifications
+-----------------------
+
+Notifications in StoryBoard are handled by workers which subscribe to
+events on a message queue. Currently only RabbitMQ is supported. The
+docker-compose.yml file runs a RabbitMQ server, and the provided config
+file is already set up to enable notifications.
+
+To run the workers so that notifications are actually created, use tox
+in the root of the ``storyboard`` repository::
+
+ tox -e storyboard-worker-daemon --config-file ./docker/storyboard.conf
+
+This will start 5 workers to listen for events and create any relevant
+notifications.
+
Installing and Upgrading the API server
=======================================
diff --git a/docker/.env b/docker/.env
new file mode 100644
index 00000000..d0fe50bc
--- /dev/null
+++ b/docker/.env
@@ -0,0 +1 @@
+COMPOSE_PROJECT_NAME=storyboard-dev-env
\ No newline at end of file
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
new file mode 100644
index 00000000..9493811d
--- /dev/null
+++ b/docker/docker-compose.yml
@@ -0,0 +1,26 @@
+version: '3'
+services:
+ mysql:
+ image: "mysql:latest"
+ ports:
+ - "3306:3306"
+ environment:
+ - MYSQL_DATABASE=storyboard
+ - MYSQL_ROOT_PASSWORD=insecure
+ volumes:
+ - "./mysql:/var/lib/mysql"
+
+ rabbitmq:
+ image: "rabbitmq:3"
+ ports:
+ - "5672:5672"
+ environment:
+ - RABBITMQ_DEFAULT_USER=storyboard
+ - RABBITMQ_DEFAULT_PASS=storyboard
+
+ swift:
+ image: "bouncestorage/swift-aio"
+ ports:
+ - "8888:8080"
+ volumes:
+ - "./swift:/swift/nodes"
diff --git a/docker/storyboard.conf b/docker/storyboard.conf
new file mode 100644
index 00000000..c0f1f1e7
--- /dev/null
+++ b/docker/storyboard.conf
@@ -0,0 +1,204 @@
+[DEFAULT]
+# Default log level is INFO
+# verbose and debug has the same result.
+# One of them will set DEBUG log level output
+# debug = False
+# verbose = False
+
+# Where to store lock files
+lock_path = $state_path/lock
+
+# Storyboard's working directory. Please ensure that the storyboard user has
+# read/write access to this directory.
+# working_directory = ~/.storyboard
+
+# log_format = %(asctime)s %(levelname)8s [%(name)s] %(message)s
+# log_date_format = %Y-%m-%d %H:%M:%S
+
+# use_syslog -> syslog
+# log_file and log_dir -> log_dir/log_file
+# (not log_file) and log_dir -> log_dir/{binary_name}.log
+# use_stderr -> stderr
+# (not user_stderr) and (not log_file) -> stdout
+# publish_errors -> notification system
+
+# use_syslog = False
+# syslog_log_facility = LOG_USER
+
+# use_stderr = True
+# log_file =
+# log_dir =
+
+# publish_errors = False
+
+# Address to bind the API server
+# bind_host = 0.0.0.0
+
+# Port the bind the API server to
+# bind_port = 8080
+
+# Enable notifications. This feature drives deferred processing, reporting,
+# and subscriptions.
+enable_notifications = True
+
+# Enable editing/deletion of comments. When enabled, users can edit their own
+# comments and admins can delete comments.
+# enable_editable_comments = True
+
+[oauth]
+# StoryBoard's oauth configuration.
+
+# OpenId Authentication endpoint
+# openid_url = https://login.launchpad.net/+openid
+
+# Time in seconds before an authorization code expires.
+# authorization_code_ttl = 300
+
+# Time in seconds before an access_token expires
+# access_token_ttl = 3600
+
+# Time in seconds before an refresh_token expires
+# refresh_token_ttl = 604800
+
+# A list of valid client id's that may connect to StoryBoard.
+# valid_oauth_clients = storyboard.openstack.org, localhost
+
+[scheduler]
+# Storyboard's scheduled task management configuration
+
+# Enable or disable scheduling (Default disabled)
+# enable = true
+
+[cors]
+# W3C CORS configuration. For more information, see http://www.w3.org/TR/cors/
+
+# List of permitted CORS domains.
+allowed_origins = https://storyboard.openstack.org, http://localhost:9000
+
+# CORS browser options cache max age (in seconds)
+# max_age=3600
+
+[notifications]
+
+# Host of the rabbitmq server.
+rabbit_host=localhost
+
+# The RabbitMQ login method
+rabbit_login_method = AMQPLAIN
+
+# The RabbitMQ userid.
+rabbit_userid = storyboard
+
+# The RabbitMQ password.
+rabbit_password = storyboard
+
+# The RabbitMQ broker port where a single node is used.
+rabbit_port = 5672
+
+# The virtual host within which our queues and exchanges live.
+rabbit_virtual_host = /
+
+# Application name that binds to rabbit.
+rabbit_application_name=storyboard
+
+# The name of the topic exchange to which storyboard will broadcast its events.
+rabbit_exchange_name=storyboard
+
+# The name of the queue that will be created for API events.
+rabbit_event_queue_name=storyboard_events
+
+# The number of connection attempts before giving-up
+rabbit_connection_attempts = 6
+
+# The interval between connection attempts (in seconds)
+rabbit_retry_delay = 10
+
+[database]
+# This line MUST be changed to actually run storyboard
+# Example:
+connection = mysql+pymysql://root:insecure@127.0.0.1:3306/storyboard?charset=utf8mb4
+# Replace 127.0.0.1 above with the IP address of the database used by the
+# main storyboard server. (Leave it as is if the database runs on this host.)
+# connection=sqlite://
+
+# The SQLAlchemy connection string used to connect to the slave database
+# slave_connection =
+
+# Database reconnection retry times - in event connectivity is lost
+# set to -1 implies an infinite retry count
+# max_retries = 10
+
+# Database reconnection interval in seconds - if the initial connection to the
+# database fails
+# retry_interval = 10
+
+# Minimum number of SQL connections to keep open in a pool
+# min_pool_size = 1
+
+# Maximum number of SQL connections to keep open in a pool
+# max_pool_size = 10
+
+# Timeout in seconds before idle sql connections are reaped
+# idle_timeout = 3600
+
+# If set, use this value for max_overflow with sqlalchemy
+# max_overflow = 20
+
+# Verbosity of SQL debugging information. 0=None, 100=Everything
+# connection_debug = 0
+
+# Add python stack traces to SQL as comment strings
+# connection_trace = False
+
+# If set, use this value for pool_timeout with sqlalchemy
+# pool_timeout = 10
+
+[plugin_token_cleaner]
+# Enable/Disable the periodic token cleaner plugin. This requires scheduled
+# management to be enabled.
+# enable = True
+
+[plugin_email]
+# Enable, or disable, the notification email plugin.
+# enable = True
+
+# The email address from which storyboard will send its messages.
+# sender = StoryBoard (Do Not Reply) <do_not_reply@storyboard.openstack.org>
+
+# The email address of the Reply-To header (optional).
+# reply_to =
+
+# The default url base to use in emails, if Referer is not set.
+# default_url = https://storyboard.openstack.org/
+
+# The SMTP server to use.
+# smtp_host = localhost
+
+# The SMTP Server Port to connect to (default 25).
+# smtp_port = 25
+
+# The SMTP socket timeout, in seconds
+# smtp_timeout = 10
+
+# The FQDN of the sending host when identifying itself to the SMTP server
+# (optional).
+# smtp_local_hostname =
+
+# Path to the SSL Keyfile, when using ESMTP. Please make sure the storyboard
+# client can read this file.
+# smtp_ssl_keyfile =
+
+# Path to the SSL Certificate, when using ESMTP. Please make sure the
+# storyboard client can read this file.
+# smtp_ssl_certfile =
+
+# Username/login for the SMTP server.
+# smtp_user =
+
+# Password for the SMTP server.
+# smtp_password =
+
+[attachments]
+
+enable_attachments = True
+storage_backend = swift