recordsansible/ara-server
Code Issues Proposed changes

Refactor configuration to use dynaconf

Browse Source 
This changes the configuration engine from everett to dynaconf.
dynaconf allows loading configuration from files (json, ini, yaml, toml)
as well as environment variables prefixed by ARA_.

Our usage of dynaconf is similar to the use case from the Pulp [1]
project and they have documented an issue when loading database
parameters [2]. This issue is worked around by importing dynaconf in the
different entry points.

This introduces some other changes as well:
- We're now creating a default configuration and data directory at
  ~/.ara. The location of this directory is controlled with the
  ARA_BASE_DIR environment variable.
- We're now creating a default configuration template in
  ~/.ara/default_config.yaml.
- The default database is now located at ~/.ara/ara.sqlite. The location
  of this database can be customized with the ARA_DATABASE_NAME
  environment variable.
  Note that ARA 0.x used "~/.ara/ansible.sqlite" -- the file name change
  is deliberate in order to avoid user databases clashing between
  versions.

More documentation on this will be available in an upcoming patch.

[1]: https://github.com/pulp/pulp
[2]: https://github.com/rochacbruno/dynaconf/issues/89

Change-Id: I8178b4ca9f2b4d7f4c45c296c08391e84e8b990d
This commit is contained in:
David Moreau Simard 2018-12-13 14:47:31 -05:00 committed by Florian Apolloner
parent 6c00d7552b
commit 16aa41eaf8
10 changed files with 118 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
ara/server/__main__.py
View File

@ -4,10 +4,9 @@ import sys
def main():
from ara import server
os.environ.setdefault("ARA_CFG", os.path.dirname(server.__file__) + "/configs/dev.cfg")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ara.server.settings")
# https://github.com/rochacbruno/dynaconf/issues/89
from dynaconf.contrib import django_dynaconf # noqa
from django.core.management import execute_from_command_line

3
ara/server/configs/dev.cfg
View File

@ -1,3 +0,0 @@
[ara]
debug = true
secret_key = dev

5
ara/server/configs/integration.cfg
View File

@ -1,5 +0,0 @@
[ara]
debug = true
log_level = DEBUG
secret_key = integration
allowed_hosts = localhost

2
ara/server/configs/test.cfg
View File

@ -1,2 +0,0 @@
[ara]
secret_key = test

162
ara/server/settings.py
View File

@ -1,20 +1,47 @@
import os
import sys
import textwrap
from .utils import EverettEnviron
import yaml
env = EverettEnviron(DEBUG=(bool, False))
# Ensure default base configuration/data directory exists
BASE_DIR = os.environ.get("ARA_BASE_DIR", os.path.expanduser("~/.ara"))
SERVER_DIR = os.path.join(BASE_DIR, "server")
if not os.path.isdir(SERVER_DIR):
os.makedirs(SERVER_DIR, mode=0o700)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
SECRET_KEY = env("SECRET_KEY")
DEBUG = env.bool("DEBUG", default=False)
ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", default=[])
# Django built-in server and npm development server
ALLOWED_HOSTS = ["127.0.0.1", "localhost"]
CORS_ORIGIN_WHITELIST = ["127.0.0.1:8000", "localhost:3000"]
CORS_ORIGIN_ALLOW_ALL = True
ADMINS = ()
# Dynaconf Configuration
SECRET_KEY = True
GLOBAL_ENV_FOR_DYNACONF = "ARA"
ENVVAR_FOR_DYNACONF = "ARA_SETTINGS"
SETTINGS_MODULE_FOR_DYNACONF = "ara.server.settings"
# We're not expecting ARA to use multiple concurrent databases.
# Make it easier for users to specify the configuration for a single database.
DATABASE_ENGINE = os.environ.get("ARA_DATABASE_ENGINE", "django.db.backends.sqlite3")
DATABASE_NAME = os.environ.get("ARA_DATABASE_NAME", os.path.join(SERVER_DIR, "ansible.sqlite"))
DATABASE_USER = os.environ.get("ARA_DATABASE_USER", None)
DATABASE_PASSWORD = os.environ.get("ARA_DATABASE_PASSWORD", None)
DATABASE_HOST = os.environ.get("ARA_DATABASE_HOST", None)
DATABASE_PORT = os.environ.get("ARA_DATABASE_PORT", None)
DATABASES = {
"default": {
"ENGINE": DATABASE_ENGINE,
"NAME": DATABASE_NAME,
"USER": DATABASE_USER,
"PASSWORD": DATABASE_PASSWORD,
"HOST": DATABASE_HOST,
"PORT": DATABASE_PORT,
}
}
INSTALLED_APPS = [
"django.contrib.auth",
"django.contrib.contenttypes",
@ -39,15 +66,6 @@ MIDDLEWARE = [
"django.middleware.clickjacking.XFrameOptionsMiddleware",
]
# TODO: Only needed in dev?
CORS_ORIGIN_ALLOW_ALL = True
# Django built-in server and npm development server
CORS_ORIGIN_WHITELIST = ("127.0.0.1:8000", "localhost:3000")
ROOT_URLCONF = "ara.server.urls"
APPEND_SLASH = False
TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
@ -64,10 +82,6 @@ TEMPLATES = [
}
]
WSGI_APPLICATION = "ara.server.wsgi.application"
DATABASES = {"default": env.db(default="sqlite:///%s" % os.path.join(BASE_DIR, "db.sqlite3"))}
AUTH_PASSWORD_VALIDATORS = [
{"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"},
{"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"},
@ -75,51 +89,22 @@ AUTH_PASSWORD_VALIDATORS = [
{"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"},
]
LANGUAGE_CODE = "en-us"
USE_TZ = True
TIME_ZONE = "UTC"
USE_I18N = True
USE_L10N = True
USE_TZ = True
LANGUAGE_CODE = "en-us"
STATIC_URL = "/static/"
STATIC_ROOT = os.path.join(BASE_DIR, "www", "static")
STATIC_ROOT = os.path.join(SERVER_DIR, "www", "static")
MEDIA_URL = "/media/"
MEDIA_ROOT = os.path.join(SERVER_DIR, "www", "media")
MEDIA_ROOT = os.path.join(BASE_DIR, "www", "media")
# fmt: off
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {"normal": {"format": "%(asctime)s %(levelname)s %(name)s: %(message)s"}},
"handlers": {
"console": {
"class": "logging.StreamHandler",
"formatter": "normal",
"level": env("LOG_LEVEL", default="INFO"),
"stream": sys.stdout,
}
},
"loggers": {
"ara": {
"handlers": ["console"],
"level": env("LOG_LEVEL", default="INFO"),
"propagate": 0
}
},
"root": {
"handlers": ["console"],
"level": env("LOG_LEVEL", default="DEBUG")
},
}
# fmt: on
WSGI_APPLICATION = "ara.server.wsgi.application"
ROOT_URLCONF = "ara.server.urls"
APPEND_SLASH = False
REST_FRAMEWORK = {
"DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination",
"PAGE_SIZE": 100,
@ -135,3 +120,62 @@ REST_FRAMEWORK = {
),
"TEST_REQUEST_DEFAULT_FORMAT": "json",
}
DEBUG = False
LOG_LEVEL = "INFO"
# fmt: off
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {"normal": {"format": "%(asctime)s %(levelname)s %(name)s: %(message)s"}},
"handlers": {
"console": {
"class": "logging.StreamHandler",
"formatter": "normal",
"level": LOG_LEVEL,
"stream": "ext://sys.stdout",
}
},
"loggers": {
"ara": {
"handlers": ["console"],
"level": LOG_LEVEL,
"propagate": 0
}
},
"root": {
"handlers": ["console"],
"level": LOG_LEVEL
},
}
# fmt: on
# TODO: Split this out to a CLI command (django-admin command ?)
DEFAULT_CONFIG = os.path.join(SERVER_DIR, "default_config.yaml")
if not os.path.exists(DEFAULT_CONFIG):
CONFIG = dict(
BASE_DIR=BASE_DIR,
ALLOWED_HOSTS=ALLOWED_HOSTS,
CORS_ORIGIN_WHITELIST=CORS_ORIGIN_WHITELIST,
CORS_ORIGIN_ALLOW_ALL=CORS_ORIGIN_ALLOW_ALL,
SECRET_KEY="please-change-this",
DATABASES=DATABASES,
STATIC_URL=STATIC_URL,
STATIC_ROOT=STATIC_ROOT,
MEDIA_URL=MEDIA_URL,
MEDIA_ROOT=MEDIA_ROOT,
DEBUG=DEBUG,
LOG_LEVEL=LOG_LEVEL,
LOGGING=LOGGING,
)
with open(DEFAULT_CONFIG, "w+") as config_file:
comment = f"""
---
# This is a default configuration template generated by ARA.
# To use a configuration file such as this one, you need to export the
# ARA_SETTINGS configuration variable like so:
# $ export ARA_SETTINGS={DEFAULT_CONFIG}
"""
config_file.write(textwrap.dedent(comment))
yaml.dump({"default": CONFIG}, config_file, default_flow_style=False)

89
ara/server/utils.py
View File

@ -1,89 +0,0 @@
import os
import environ
from configobj import ConfigObj
from django.core.exceptions import ImproperlyConfigured
from everett import ConfigurationError
from everett.manager import ConfigEnvFileEnv, ConfigIniEnv, ConfigManager, ConfigOSEnv, listify
__all__ = ["EverettEnviron"]
class DumbConfigIniEnv(ConfigIniEnv):
"""Simple ConfigIniEnv with disabled list parsing that actually aborts after the first file."""
# TODO: Remove once upstream is fixed (https://github.com/willkg/everett/pull/71)
def __init__(self, possible_paths):
self.cfg = {}
possible_paths = listify(possible_paths)
for path in possible_paths:
if not path:
continue
path = os.path.abspath(os.path.expanduser(path.strip()))
if path and os.path.isfile(path):
self.cfg = self.parse_ini_file(path)
break
def parse_ini_file(cls, path):
cfgobj = ConfigObj(path, list_values=False)
def extract_section(namespace, d):
cfg = {}
for key, val in d.items():
if isinstance(d[key], dict):
cfg.update(extract_section(namespace + [key], d[key]))
else:
cfg["_".join(namespace + [key]).upper()] = val
return cfg
return extract_section([], cfgobj.dict())
class EnvironProxy:
def __init__(self, cfg):
self.cfg = cfg
def __contains__(self, key):
try:
self.cfg(key)
except ConfigurationError:
return False
return True
def __getitem__(self, key):
try:
return self.cfg(key, raw_value=True)
except ConfigurationError as err:
raise KeyError("Missing key %r" % key) from err
class EverettEnviron(environ.Env):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.ENVIRON = EnvironProxy(
ConfigManager(
[
ConfigOSEnv(),
ConfigEnvFileEnv(".env"),
DumbConfigIniEnv([os.environ.get("ARA_CFG"), "~/.config/ara/server.cfg", "/etc/ara/server.cfg"]),
]
).with_namespace("ara")
)
def get_value(self, var, cast=None, default=environ.Env.NOTSET, parse_default=False):
try:
return super().get_value(var, cast, default, parse_default)
except ImproperlyConfigured as e:
# Rewrite the django-environ exception to match our configs better
if default is self.NOTSET and str(e) == "Set the {0} environment variable".format(var):
error_msg = (
"Set the ARA_{0} environment variable or set {1} in the [ara] section "
"of your configuration file."
).format(var, var.lower())
raise ImproperlyConfigured(error_msg)
raise

5
ara/server/wsgi.py
View File

@ -1,7 +1,8 @@
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ara.server.settings")
# https://github.com/rochacbruno/dynaconf/issues/89
from dynaconf.contrib import django_dynaconf # noqa
from django.core.wsgi import get_wsgi_application # noqa
application = get_wsgi_application()

2
requirements.txt
View File

@ -6,4 +6,4 @@ django-cors-headers
drf-extensions
django-filter
django-environ
everett
dynaconf

2
setup.cfg
View File

@ -75,7 +75,7 @@ exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,ara/api/migrations
[isort]
known_first_party = ara
default_section = THIRDPARTY
skip = build,.git,.tox,.cache,.venv,ara/api/migrations
skip = build,.git,.tox,.cache,.venv,ara/api/migrations,ara/server/wsgi.py
not_skip = __init__.py
multi_line_output=3
include_trailing_comma=True

10
tox.ini
View File

@ -30,6 +30,10 @@ commands =
ara-manage migrate
ara-manage collectstatic --clear --no-input
ara-manage runserver
setenv =
ARA_DEBUG=true
ARA_LOG_LEVEL=DEBUG
ARA_BASE_DIR={toxinidir}/.tox/ansible-integration/tmp/ara
# Temporary venv to help bootstrap integration
[testenv:ansible-integration]
@ -37,11 +41,13 @@ deps =
git+https://git.openstack.org/openstack/ara-plugins@master#egg=ara_plugins
git+https://git.openstack.org/openstack/ara-clients@master#egg=ara_clients
commands =
rm -f {toxinidir}/db.sqlite3
rm -rf {toxinidir}/.tox/ansible-integration/tmp/ara
bash -c 'ANSIBLE_CALLBACK_PLUGINS=$(python -c "import os,ara.plugins; print(os.path.dirname(ara.plugins.__file__))")/callback ansible-playbook {toxinidir}/hacking/test-playbook.yml'
python {toxinidir}/hacking/validate.py
setenv =
ARA_CFG={toxinidir}/ara/server/configs/integration.cfg
ARA_DEBUG=true
ARA_LOG_LEVEL=DEBUG
ARA_BASE_DIR={toxinidir}/.tox/ansible-integration/tmp/ara
whitelist_externals =
rm
bash