From 4589ca62ba1e9cc23397f71ff065eda63bbf4ec9 Mon Sep 17 00:00:00 2001
From: Ilya Shakhat <ishakhat@mirantis.com>
Date: Mon, 15 Jul 2013 20:23:17 +0400
Subject: [PATCH] Stackalytics deployment clean up
* Scripts and conf locations are set in setup.cfg
* Flask-styled is replaced by oslo config
* Scripts from bin are replaced by one generated by entry_points
Change-Id: I26620b8d945d331b2d5abebce41ca177f3e93571
---
bin/dashboard | 11 ------
bin/processor | 11 ------
dashboard/web.py | 36 ++++++++++++++----
etc/dashboard.conf | 5 ---
etc/stackalytics.conf | 12 ++++--
setup.cfg | 9 ++++-
stackalytics/processor/commit_processor.py | 4 +-
stackalytics/processor/config.py | 44 ++++++++++++++++++++++
stackalytics/processor/main.py | 26 ++-----------
9 files changed, 93 insertions(+), 65 deletions(-)
delete mode 100755 bin/dashboard
delete mode 100755 bin/processor
delete mode 100644 etc/dashboard.conf
create mode 100644 stackalytics/processor/config.py
diff --git a/bin/dashboard b/bin/dashboard
deleted file mode 100755
index d7862984c..000000000
--- a/bin/dashboard
+++ /dev/null
@@ -1,11 +0,0 @@
-#!.venv/bin/python
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-import os
-import sys
-sys.path.insert(0, os.getcwd())
-
-from dashboard.web import app
-
-
-app.run()
diff --git a/bin/processor b/bin/processor
deleted file mode 100755
index 415b67946..000000000
--- a/bin/processor
+++ /dev/null
@@ -1,11 +0,0 @@
-#!.venv/bin/python
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-import os
-import sys
-sys.path.insert(0, os.getcwd())
-
-from stackalytics.processor.main import main
-
-
-main()
diff --git a/dashboard/web.py b/dashboard/web.py
index 618d77f3c..8e1216715 100644
--- a/dashboard/web.py
+++ b/dashboard/web.py
@@ -23,9 +23,12 @@ import urllib
import flask
from flask.ext import gravatar as gravatar_ext
+from oslo.config import cfg
import time
from dashboard import memory_storage
+from stackalytics.openstack.common import log as logging
+from stackalytics.processor import config
from stackalytics.processor import persistent_storage
from stackalytics.processor import runtime_storage
from stackalytics.processor import user_utils
@@ -33,10 +36,6 @@ from stackalytics.processor import user_utils
# Constants and Parameters ---------
-DEBUG = True
-RUNTIME_STORAGE_URI = 'memcached://127.0.0.1:11211'
-PERSISTENT_STORAGE_URI = 'mongodb://localhost'
-
DEFAULTS = {
'metric': 'commits',
'release': 'havana',
@@ -62,16 +61,33 @@ app = flask.Flask(__name__)
app.config.from_object(__name__)
app.config.from_envvar('DASHBOARD_CONF', silent=True)
+LOG = logging.getLogger(__name__)
+
+conf = cfg.CONF
+conf.register_opts(config.OPTS)
+logging.setup('dashboard')
+LOG.info('Logging enabled')
+
+conf_file = os.getenv('STACKALYTICS_CONF')
+if (conf_file is None) or (not os.path.isfile(conf_file)):
+ conf_file = '/etc/stackalytics/stackalytics.conf'
+
+if os.path.isfile(conf_file):
+ conf(default_config_files=[conf_file])
+ app.config['DEBUG'] = cfg.CONF.debug
+else:
+ LOG.warn('Conf file is empty or not exist')
+
def get_vault():
vault = getattr(app, 'stackalytics_vault', None)
if not vault:
vault = {}
vault['runtime_storage'] = runtime_storage.get_runtime_storage(
- RUNTIME_STORAGE_URI)
+ cfg.CONF.runtime_storage_uri)
vault['persistent_storage'] = (
persistent_storage.get_persistent_storage(
- PERSISTENT_STORAGE_URI))
+ cfg.CONF.persistent_storage_uri))
vault['memory_storage'] = memory_storage.get_memory_storage(
memory_storage.MEMORY_STORAGE_CACHED,
vault['runtime_storage'].get_update(os.getpid()))
@@ -208,7 +224,7 @@ def exception_handler():
try:
return f(*args, **kwargs)
except Exception as e:
- print e
+ LOG.debug(e)
flask.abort(404)
return decorated_function
@@ -510,5 +526,9 @@ def make_commit_message(record):
gravatar = gravatar_ext.Gravatar(app, size=100, rating='g',
default='wavatar')
+
+def main():
+ app.run(cfg.CONF.listen_host, cfg.CONF.listen_port)
+
if __name__ == '__main__':
- app.run('0.0.0.0')
+ main()
diff --git a/etc/dashboard.conf b/etc/dashboard.conf
deleted file mode 100644
index 90826e6f3..000000000
--- a/etc/dashboard.conf
+++ /dev/null
@@ -1,5 +0,0 @@
-#
-# Configuration of stackalytics dashboard
-#
-
-DEBUG = True
\ No newline at end of file
diff --git a/etc/stackalytics.conf b/etc/stackalytics.conf
index c4dcae893..bbe46bb04 100644
--- a/etc/stackalytics.conf
+++ b/etc/stackalytics.conf
@@ -3,10 +3,10 @@
# debug = False
# Default data
-# default-data = etc/default_data.json
+# default-data = /etc/stackalytics/default_data.json
# The folder that holds all project sources to analyze
-# sources_root = ../metric-root-tmp
+# sources_root = /var/run/stackalytics
# Runtime storage URI
# runtime_storage_uri = memcached://127.0.0.1:11211
@@ -21,4 +21,10 @@
# repo_poll_period = 300
# Address of update handler
-# frontend_update_address = http://user:user@localhost/update/%s
\ No newline at end of file
+# frontend_update_address = http://user:user@localhost/update/%s
+
+# Hostname where dashboard listens on
+# listen_host = 127.0.0.1
+
+# Port where dashboard listens on
+# listen_port = 8080
diff --git a/setup.cfg b/setup.cfg
index 42eeee4ab..be4db8631 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -4,7 +4,7 @@ version = 0.1
summary = OpenStack analytics dashboard
description-file =
README.rst
-author = OpenStack Stackalytics Project
+author = OpenStack
author-email = openstack-dev@lists.openstack.org
home-page = http://www.openstack.org/
classifier =
@@ -21,6 +21,10 @@ classifier =
packages =
dashboard
stackalytics
+data_files =
+ etc/stackalytics =
+ etc/stackalytics.conf
+ etc/default_data.json
[global]
setup-hooks =
@@ -28,4 +32,5 @@ setup-hooks =
[entry_points]
console_scripts =
- stackalytics-dashboard = dashboard.dashboard:main
+ stackalytics-dashboard = dashboard.web:main
+ stackalytics-processor = stackalytics.processor.main:main
diff --git a/stackalytics/processor/commit_processor.py b/stackalytics/processor/commit_processor.py
index fde5ca6ef..d1fd4e6f4 100644
--- a/stackalytics/processor/commit_processor.py
+++ b/stackalytics/processor/commit_processor.py
@@ -79,11 +79,11 @@ class CachedProcessor(CommitProcessor):
def _unknown_user_email(self, email):
lp_profile = None
- if not re.match(r'[^@]+@[^@]+\.[^@]+', email):
+ if not re.match(r'[\w\d_\.-]+@([\w\d_\.-]+\.)+[\w]+', email):
LOG.debug('User email is not valid %s' % email)
else:
LOG.debug('Lookup user email %s at Launchpad' % email)
- lp = launchpad.Launchpad.login_anonymously(cfg.CONF.launchpad_user)
+ lp = launchpad.Launchpad.login_anonymously('stackalytics')
try:
lp_profile = lp.people.getByEmail(email=email)
except Exception as error:
diff --git a/stackalytics/processor/config.py b/stackalytics/processor/config.py
new file mode 100644
index 000000000..9b020eb69
--- /dev/null
+++ b/stackalytics/processor/config.py
@@ -0,0 +1,44 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo.config import cfg
+
+OPTS = [
+ cfg.StrOpt('default-data', default='etc/default_data.json',
+ help='Default data'),
+ cfg.StrOpt('sources-root', default='/var/run/stackalytics',
+ help='The folder that holds all project sources to analyze'),
+ cfg.StrOpt('runtime-storage-uri', default='memcached://127.0.0.1:11211',
+ help='Storage URI'),
+ cfg.StrOpt('frontend-update-address',
+ default='http://user:user@localhost/update/%s',
+ help='Address of update handler'),
+ cfg.StrOpt('repo-poll-period', default='300',
+ help='Repo poll period in seconds'),
+ cfg.StrOpt('persistent-storage-uri', default='mongodb://localhost',
+ help='URI of persistent storage'),
+ cfg.BoolOpt('sync-default-data', default=False,
+ help='Update persistent storage with default data. '
+ 'Existing data is not overwritten'),
+ cfg.BoolOpt('force-sync-default-data', default=False,
+ help='Completely overwrite persistent storage with the '
+ 'default data'),
+ cfg.BoolOpt('filter-robots', default=True,
+ help='Filter out commits from robots'),
+ cfg.StrOpt('listen-host', default='127.0.0.1',
+ help='The address dashboard listens on'),
+ cfg.IntOpt('listen-port', default=8080,
+ help='The port dashboard listens on'),
+]
diff --git a/stackalytics/processor/main.py b/stackalytics/processor/main.py
index 64f416419..b44a0ae98 100644
--- a/stackalytics/processor/main.py
+++ b/stackalytics/processor/main.py
@@ -19,6 +19,7 @@ from psutil import _error
from stackalytics.openstack.common import log as logging
from stackalytics.processor import commit_processor
+from stackalytics.processor import config
from stackalytics.processor import persistent_storage
from stackalytics.processor import runtime_storage
from stackalytics.processor import vcs
@@ -26,27 +27,6 @@ from stackalytics.processor import vcs
LOG = logging.getLogger(__name__)
-OPTS = [
- cfg.StrOpt('default-data', default='etc/default_data.json',
- help='Default data'),
- cfg.StrOpt('sources-root', default=None, required=True,
- help='The folder that holds all project sources to analyze'),
- cfg.StrOpt('runtime-storage-uri', default='memcached://127.0.0.1:11211',
- help='Storage URI'),
- cfg.StrOpt('persistent-storage-uri', default='mongodb://localhost',
- help='URI of persistent storage'),
- cfg.BoolOpt('sync-default-data', default=False,
- help='Update persistent storage with default data. '
- 'Existing data is not overwritten'),
- cfg.BoolOpt('force-sync-default-data', default=False,
- help='Completely overwrite persistent storage with the '
- 'default data'),
- cfg.StrOpt('launchpad-user', default='stackalytics-bot',
- help='User to access Launchpad'),
- cfg.BoolOpt('filter-robots', default=True,
- help='Filter out commits from robots'),
-]
-
def get_pids():
uwsgi_dict = {}
@@ -107,8 +87,8 @@ def update_repos(runtime_storage, persistent_storage):
def main():
# init conf and logging
conf = cfg.CONF
- conf.register_cli_opts(OPTS)
- conf.register_opts(OPTS)
+ conf.register_cli_opts(config.OPTS)
+ conf.register_opts(config.OPTS)
conf()
logging.setup('stackalytics')