From 9ac0db3253f652ebaad7f98fc87c29b8eabab411 Mon Sep 17 00:00:00 2001 From: Vladislav Kuzmin Date: Wed, 11 Feb 2015 15:47:59 +0400 Subject: [PATCH] Add oslo.log library Add oslo.log lib in application. Move API related options from refstack/api/config.py to the oslo_config's configuration file. Make regeneration of example config file with oslo.log namespace and with moved API options. Change-Id: Id0db95fe34eb8faa284e9d716cbeebdf6d25c183 --- bin/refstack-manage | 7 +- etc/refstack.conf.sample | 115 +++++++++++++++++++++++++++++++++ refstack/api/app.py | 82 +++++++++++++++++------ refstack/api/config.py | 29 --------- refstack/api/controllers/v1.py | 5 +- refstack/db/utils.py | 6 +- refstack/opts.py | 2 + refstack/utils.py | 13 ---- requirements.txt | 3 +- tox.ini | 3 +- 10 files changed, 194 insertions(+), 71 deletions(-) diff --git a/bin/refstack-manage b/bin/refstack-manage index 073c377d..50f66e72 100755 --- a/bin/refstack-manage +++ b/bin/refstack-manage @@ -20,12 +20,16 @@ Command-line utility for database manage import sys -from oslo.config import cfg +from oslo_config import cfg +from oslo_log import log from refstack.db import migration +LOG = log.getLogger(__name__) CONF = cfg.CONF +log.register_options(CONF) + class DatabaseManager(object): @@ -91,4 +95,5 @@ CONF.register_cli_opt(command_opt) if __name__ == '__main__': CONF(sys.argv[1:], project='refstack') + log.setup(CONF, 'refstack') CONF.command.func() diff --git a/etc/refstack.conf.sample b/etc/refstack.conf.sample index 70342478..7d2913d1 100644 --- a/etc/refstack.conf.sample +++ b/etc/refstack.conf.sample @@ -1,5 +1,91 @@ [DEFAULT] +# +# From oslo.log +# + +# Print debugging output (set logging level to DEBUG instead of +# default WARNING level). (boolean value) +#debug = false + +# Print more verbose output (set logging level to INFO instead of +# default WARNING level). (boolean value) +#verbose = false + +# The name of a logging configuration file. This file is appended to +# any existing logging configuration files. For details about logging +# configuration files, see the Python logging module documentation. +# (string value) +# Deprecated group/name - [DEFAULT]/log_config +#log_config_append = + +# DEPRECATED. A logging.Formatter log message format string which may +# use any of the available logging.LogRecord attributes. This option +# is deprecated. Please use logging_context_format_string and +# logging_default_format_string instead. (string value) +#log_format = + +# Format string for %%(asctime)s in log records. Default: %(default)s +# . (string value) +#log_date_format = %Y-%m-%d %H:%M:%S + +# (Optional) Name of log file to output to. If no default is set, +# logging will go to stdout. (string value) +# Deprecated group/name - [DEFAULT]/logfile +#log_file = + +# (Optional) The base directory used for relative --log-file paths. +# (string value) +# Deprecated group/name - [DEFAULT]/logdir +#log_dir = + +# Use syslog for logging. Existing syslog format is DEPRECATED during +# I, and will change in J to honor RFC5424. (boolean value) +#use_syslog = false + +# (Optional) Enables or disables syslog rfc5424 format for logging. If +# enabled, prefixes the MSG part of the syslog message with APP-NAME +# (RFC5424). The format without the APP-NAME is deprecated in I, and +# will be removed in J. (boolean value) +#use_syslog_rfc_format = false + +# Syslog facility to receive log lines. (string value) +#syslog_log_facility = LOG_USER + +# Log output to standard error. (boolean value) +#use_stderr = true + +# Format string to use for log messages with context. (string value) +#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s + +# Format string to use for log messages without context. (string +# value) +#logging_default_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [-] %(instance)s%(message)s + +# Data to append to log format when level is DEBUG. (string value) +#logging_debug_format_suffix = %(funcName)s %(pathname)s:%(lineno)d + +# Prefix each line of exception output with this format. (string +# value) +#logging_exception_prefix = %(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s + +# List of logger=LEVEL pairs. (list value) +#default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN + +# Enables or disables publication of error events. (boolean value) +#publish_errors = false + +# Enables or disables fatal status of deprecations. (boolean value) +#fatal_deprecations = false + +# The format for an instance that is passed with the log message. +# (string value) +#instance_format = "[instance: %(uuid)s] " + +# The format for an instance UUID that is passed with the log message. +# (string value) +#instance_uuid_format = "[instance: %(uuid)s] " + # # From refstack # @@ -8,6 +94,35 @@ #db_backend = sqlalchemy +[api] + +# +# From refstack +# + +# The directory where your static files can be found. Pecan comes with +# middleware that can be used to serve static files (like CSS and +# Javascript files) during development. %(project_root)s is special +# variable that point to the root directory of Refstack project. Value +# of this option must contain %(project_root)s variable. Directory +# with static files specified relative the project root. (string +# value) +#static_root = %(project_root)s/static + +# Points to the directory where your template files live. +# %(project_root)s is special variable that point to the root +# directory of Refstack project. Value of this option must contain +# %(project_root)s variable. Directory with template files specified +# relative the project root. (string value) +#template_path = %(project_root)s/templates + +# Switch Refstack app into debug mode. Helpful for development. In +# debug mode static file will be served by pecan application. Also, +# server responses will contain some details with debug information. +# (boolean value) +#app_dev_mode = false + + [database] # diff --git a/refstack/api/app.py b/refstack/api/app.py index 1b1c8f59..b60576a3 100644 --- a/refstack/api/app.py +++ b/refstack/api/app.py @@ -20,24 +20,64 @@ import logging import os from oslo_config import cfg +from oslo_log import log +from oslo_log import loggers import pecan from pecan import hooks import webob -from refstack import utils +LOG = log.getLogger(__name__) -logger = logging.getLogger(__name__) +PROJECT_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), + os.pardir) + +API_OPTS = [ + cfg.StrOpt('static_root', + default='%(project_root)s/static', + help='The directory where your static files can ' + 'be found. Pecan comes with middleware that can be used ' + 'to serve static files (like CSS and Javascript files) ' + 'during development. %(project_root)s is special variable ' + 'that point to the root directory of Refstack project. ' + 'Value of this option must contain %(project_root)s ' + 'variable. Directory with static files specified relative ' + 'the project root.' + ), + cfg.StrOpt('template_path', + default='%(project_root)s/templates', + help='Points to the directory where your template files live. ' + '%(project_root)s is special variable that point to the ' + 'root directory of Refstack project. Value of this option ' + 'must contain %(project_root)s variable. Directory with ' + 'template files specified relative the project root.' + ), + cfg.BoolOpt('app_dev_mode', + default=False, + help='Switch Refstack app into debug mode. Helpful for ' + 'development. In debug mode static file will be served ' + 'by pecan application. Also, server responses will ' + 'contain some details with debug information.' + ), +] CONF = cfg.CONF +opt_group = cfg.OptGroup(name='api', + title='Options for the Refstack API') + +CONF.register_group(opt_group) +CONF.register_opts(API_OPTS, opt_group) + +log.register_options(CONF) + class JSONErrorHook(hooks.PecanHook): """ A pecan hook that translates webob HTTP errors into a JSON format. """ - def __init__(self, app_config): + def __init__(self): """Hook init.""" - self.debug = app_config.get('debug', False) + self.debug = CONF.api.app_dev_mode def on_error(self, state, exc): """Request error handler.""" @@ -52,7 +92,7 @@ class JSONErrorHook(hooks.PecanHook): content_type='application/json' ) else: - logger.exception(exc) + LOG.exception(exc) body = {'code': 500, 'title': 'Internal Server Error'} if self.debug: @@ -66,18 +106,6 @@ class JSONErrorHook(hooks.PecanHook): def setup_app(config): """App factory.""" - app_conf = dict(config.app) - - app = pecan.make_app( - app_conf.pop('root'), - logging=getattr(config, 'logging', {}), - hooks=[JSONErrorHook(app_conf), hooks.RequestViewerHook( - {'items': ['status', 'method', 'controller', 'path', 'body']}, - headers=False, writer=utils.LogWriter(logger, logging.DEBUG) - )], - **app_conf - ) - # By default we expect path to oslo config file in environment variable # REFSTACK_OSLO_CONFIG (option for testing and development) # If it is empty we look up those config files @@ -86,14 +114,30 @@ def setup_app(config): # ~/ # /etc/${project}/ # /etc/ + default_config_files = ((os.getenv('REFSTACK_OSLO_CONFIG'), ) if os.getenv('REFSTACK_OSLO_CONFIG') else cfg.find_config_files('refstack')) - CONF('', project='refstack', default_config_files=default_config_files) - CONF.log_opt_values(logger, logging.DEBUG) + log.setup(CONF, 'refstack') + CONF.log_opt_values(LOG, logging.DEBUG) + + template_path = CONF.api.template_path % {'project_root': PROJECT_ROOT} + static_root = CONF.api.static_root % {'project_root': PROJECT_ROOT} + + app_conf = dict(config.app) + app = pecan.make_app( + app_conf.pop('root'), + debug=CONF.api.app_dev_mode, + static_root=static_root, + template_path=template_path, + hooks=[JSONErrorHook(), hooks.RequestViewerHook( + {'items': ['status', 'method', 'controller', 'path', 'body']}, + headers=False, writer=loggers.WritableLogger(LOG, logging.DEBUG) + )] + ) return app diff --git a/refstack/api/config.py b/refstack/api/config.py index 5fa9b583..a80655f1 100644 --- a/refstack/api/config.py +++ b/refstack/api/config.py @@ -34,33 +34,4 @@ server = { app = { 'root': 'refstack.api.controllers.root.RootController', 'modules': ['refstack.api'], - 'static_root': '%(confdir)s/../static', - 'template_path': '%(confdir)s/../templates', - # The 'debug' option should be false in production servers, but needs to be - # true in development in order to allow the static_root option to work. - 'debug': False, - 'errors': { - '404': '/error/404', - '__force_dict__': True - } -} - -logging = { - 'loggers': { - 'root': {'level': 'INFO', 'handlers': ['console']}, - 'refstack': {'level': 'DEBUG', 'handlers': ['console']} - }, - 'handlers': { - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'simple' - } - }, - 'formatters': { - 'simple': { - 'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]' - '[%(threadName)s] %(message)s') - } - } } diff --git a/refstack/api/controllers/v1.py b/refstack/api/controllers/v1.py index 549d1f0c..7e261c38 100644 --- a/refstack/api/controllers/v1.py +++ b/refstack/api/controllers/v1.py @@ -14,15 +14,14 @@ # under the License. """Version 1 of the API.""" -import logging - +from oslo_log import log import pecan from pecan import rest from refstack import db from refstack.common import validators -logger = logging.getLogger(__name__) +LOG = log.getLogger(__name__) class RestControllerWithValidation(rest.RestController): diff --git a/refstack/db/utils.py b/refstack/db/utils.py index c952b737..31dd4fa2 100644 --- a/refstack/db/utils.py +++ b/refstack/db/utils.py @@ -14,13 +14,11 @@ # under the License. """Utilities for database.""" - -import logging - from oslo_config import cfg +from oslo_log import log CONF = cfg.CONF -LOG = logging.getLogger(__name__) +LOG = log.getLogger(__name__) class PluggableBackend(object): diff --git a/refstack/opts.py b/refstack/opts.py index 6afa88d5..2e0f5f51 100644 --- a/refstack/opts.py +++ b/refstack/opts.py @@ -31,6 +31,7 @@ ] ... """ +import refstack.api.app import refstack.db.api @@ -38,4 +39,5 @@ def list_opts(): return [ # Keep a list in alphabetical order ('DEFAULT', refstack.db.api.db_opts), + ('api', refstack.api.app.API_OPTS), ] diff --git a/refstack/utils.py b/refstack/utils.py index 3692edc7..3fef52b1 100755 --- a/refstack/utils.py +++ b/refstack/utils.py @@ -20,7 +20,6 @@ from datetime import datetime import os import random -import re import string @@ -61,18 +60,6 @@ SEX_TYPE = { STRING_LEN = 64 -class LogWriter(object): - """Stream-like API to logger""" - - def __init__(self, logger, level): - self.logger = logger - self.level = level - - def write(self, s): - if re.sub('[\n ]', '', s): - self.logger.log(self.level, '\n' + s) - - def get_current_time(): return datetime.utcnow() diff --git a/requirements.txt b/requirements.txt index 87ab01d3..7ce0954c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,9 @@ alembic==0.5.0 gunicorn==18 oslo.config>=1.6.0 # Apache-2.0 oslo.db>=1.4.1 # Apache-2.0 +oslo.log pecan>=0.8.2 pyOpenSSL==0.13 pycrypto==2.6 requests==1.2.3 -jsonschema>=2.0.0,<3.0.0 \ No newline at end of file +jsonschema>=2.0.0,<3.0.0 diff --git a/tox.ini b/tox.ini index a0edcf9b..8236eab9 100644 --- a/tox.ini +++ b/tox.ini @@ -40,7 +40,8 @@ distribute = false commands = oslo-config-generator --output-file etc/refstack.conf.sample \ --namespace refstack \ - --namespace oslo.db + --namespace oslo.db \ + --namespace oslo.log [testenv:venv] commands = {posargs}