Rework tox unit tests config

* Upgrade tox to a newer version (a least 3.18.0 like nova)
* Stop using run_tests.sh (use prepare_db.sh and stestr directly)
* Add venv and mistral.log in gitignore

Change-Id: I71c206aadc4133932fb591376b020a39a78d837d
Signed-off-by: Arnaud M <arnaud.morin@gmail.com>

.gitignore

@@ -9,6 +9,7 @@
 dist
 build
 .venv
+venv
 eggs
 parts
 bin
@@ -31,6 +32,7 @@ cover/*
 .testrepository/
 .stestr/
 subunit.log
+mistral.log
 .mistral.conf
 AUTHORS
 ChangeLog

tools/prepare_db.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+set -eu
+
+db_type=$1
+
+function setup_db {
+    case ${db_type} in
+        sqlite )
+            rm -f tests.sqlite
+            ;;
+        postgresql | mysql )
+            dbname="openstack_citest"
+            username="openstack_citest"
+            password="openstack_citest"
+            ;;
+    esac
+}
+
+function setup_db_pylib {
+    case ${db_type} in
+        postgresql )
+            echo "Installing python library for PostgreSQL."
+            pip install psycopg2==2.8.3
+            ;;
+        mysql )
+            echo "Installing python library for MySQL"
+            pip install PyMySQL
+            ;;
+    esac
+}
+
+function setup_db_cfg {
+    case ${db_type} in
+        sqlite )
+            rm -f .mistral.conf
+            ;;
+        postgresql )
+            oslo-config-generator --config-file \
+                ./tools/config/config-generator.mistral.conf \
+                --output-file .mistral.conf
+            sed -i "s/#connection = <None>/connection = postgresql:\/\/$username:$password@localhost\/$dbname/g" .mistral.conf
+            ;;
+        mysql )
+            oslo-config-generator --config-file \
+                ./tools/config/config-generator.mistral.conf \
+                --output-file .mistral.conf
+            sed -i "s/#connection = <None>/connection = mysql+pymysql:\/\/$username:$password@localhost\/$dbname/g" .mistral.conf
+            ;;
+    esac
+}
+
+function upgrade_db {
+    case ${db_type} in
+        postgresql | mysql )
+            mistral-db-manage --config-file .mistral.conf upgrade head
+            ;;
+    esac
+}
+
+
+setup_db
+setup_db_pylib
+setup_db_cfg
+upgrade_db

tox.ini

@@ -1,121 +1,136 @@
 [tox]
 envlist = py3,pep8
-minversion = 2.0
+minversion = 3.18.0
-ignore_basepython_conflict = True
 
+# This is the default env for tests, which is going to be used for all py3*
 [testenv]
-basepython = python3
+description =
+  Run unit tests.
 usedevelop = True
-install_command = pip install {opts} {packages}
+install_command = python -I -m pip install -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} {opts} {packages}
-setenv = VIRTUAL_ENV={envdir}
+setenv =
-         PYTHONDONTWRITEBYTECODE = 1
+  VIRTUAL_ENV={envdir}
-         PYTHONWARNINGS=default::DeprecationWarning
+  LANGUAGE=en_US
-passenv =
+  LC_ALL=en_US.utf-8
-    http_proxy
+  OS_STDOUT_CAPTURE=1
-    HTTP_PROXY
+  OS_STDERR_CAPTURE=1
-    https_proxy
+  OS_TEST_TIMEOUT=160
-    HTTPS_PROXY
+  PYTHONDONTWRITEBYTECODE=1
-    no_proxy
-    NO_PROXY
 deps =
-  -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
   -r{toxinidir}/test-requirements.txt
   -r{toxinidir}/requirements.txt
-  # javascript engine
+allowlist_externals =
-  py_mini_racer
+  bash
 commands =
-  rm -f .testrepository/times.dbm
+  bash {toxinidir}/tools/prepare_db.sh sqlite
-  find . -type f -name "*.pyc" -delete
+  stestr run --color {posargs}
-  stestr run --slowest {posargs}
+  stestr slowest
-allowlist_externals =
-  rm
-  find
-
-[testenv:unit-postgresql]
-setenv = VIRTUAL_ENV={envdir}
-passenv = ZUUL_PROJECT
-allowlist_externals =
-  bash
-commands = bash run_tests.sh -N --db-type postgresql
-
-[testenv:unit-sqlite]
-setenv = VIRTUAL_ENV={envdir}
-passenv = ZUUL_PROJECT
-allowlist_externals =
-  bash
-commands = bash run_tests.sh -N --db-type sqlite
-
-[testenv:unit-mysql]
-setenv = VIRTUAL_ENV={envdir}
-passenv = ZUUL_PROJECT
-allowlist_externals =
-  bash
-commands = bash run_tests.sh -N --db-type mysql
 
 [testenv:pep8]
+description =
+  Run pep8 tests.
 commands =
   doc8 doc/source
   flake8 {posargs} . {toxinidir}/tools/sync_db.py
 
+# Deprecated
+[testenv:unit-postgresql]
+description =
+  Run unit tests with postgresql backend.
+commands =
+  bash {toxinidir}/tools/prepare_db.sh postgresql
+  stestr run --color {posargs}
+  stestr slowest
+
+# Deprecated
+[testenv:unit-mysql]
+description =
+  Run unit tests with mysql backend.
+commands =
+  bash {toxinidir}/tools/prepare_db.sh mysql
+  stestr run --color {posargs}
+  stestr slowest
+
 [testenv:cover]
+description =
+  Run coverage tests.
 setenv =
     {[testenv]setenv}
     PYTHON=coverage run --source mistral --parallel-mode
 commands =
+  coverage erase
   stestr run {posargs}
   coverage combine
   coverage html -d cover
   coverage xml -o cover/coverage.xml
+  coverage report
 
 [testenv:genconfig]
+description =
+  Build mistral.conf sample file.
 commands =
-    oslo-config-generator --config-file tools/config/config-generator.mistral.conf \
+  oslo-config-generator \
-    --output-file etc/mistral.conf.sample
+  --config-file=tools/config/config-generator.mistral.conf \
+  --output-file=etc/mistral.conf.sample
 
 [testenv:genpolicy]
+description =
+  Build policy.yaml sample file.
 commands =
-    oslopolicy-sample-generator --config-file tools/config/policy-generator.mistral.conf \
+  oslopolicy-sample-generator \
-    --output-file etc/policy.yaml.sample
+  --config-file=tools/config/policy-generator.mistral.conf \
+  --output-file=etc/policy.yaml.sample
 
-#set PYTHONHASHSEED=0 to prevent wsmeext.sphinxext from randomly failing.
 [testenv:venv]
-setenv = PYTHONHASHSEED=0
-commands = {posargs}
-
-#set PYTHONHASHSEED=0 to prevent wsmeext.sphinxext from randomly failing.
-[testenv:docs]
 deps =
-  -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+  {[testenv]deps}
+  -r{toxinidir}/doc/requirements.txt
+commands =
+  {posargs}
+
+[testenv:docs]
+description =
+  Build main documentation.
+deps =
   -r{toxinidir}/doc/requirements.txt
   -r{toxinidir}/requirements.txt
-setenv = PYTHONHASHSEED=0
+allowlist_externals =
+  rm
 commands =
   rm -rf doc/build
   sphinx-build -E -W --keep-going -b html doc/source doc/build/html
 
 [testenv:pdf-docs]
-deps =
+description =
-  -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+  Build PDF documentation.
-  -r{toxinidir}/doc/requirements.txt
+deps = {[testenv:docs]deps}
 allowlist_externals =
+  rm
   make
 commands =
+  rm -rf doc/build/pdf
   sphinx-build -W -b latex doc/source doc/build/pdf
   make -C doc/build/pdf
 
 [testenv:releasenotes]
-commands =
+description =
-  rm -rf releasenotes/build
+  Generate release notes.
-  sphinx-build -a -E -W -d releasenotes/build/doctrees --keep-going -b html releasenotes/source releasenotes/build/html
+deps = {[testenv:docs]deps}
-
-[testenv:api-ref]
-# This environment is called from CI scripts to test and publish
-# the API Ref to docs.openstack.org.
-commands =
-  rm -rf api-ref/build
-  sphinx-build -W --keep-going -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html
 allowlist_externals =
   rm
+commands =
+  rm -rf releasenotes/build
+  sphinx-build -W --keep-going -b html -j auto releasenotes/source releasenotes/build/html
+
+[testenv:api-ref]
+description =
+  Generate the API ref. Called from CI scripts to test and publish to docs.openstack.org.
+deps = {[testenv:docs]deps}
+allowlist_externals =
+  rm
+commands =
+  rm -rf api-ref/build
+  sphinx-build -W --keep-going -b html -j auto api-ref/source api-ref/build/html
 
 #Skip PEP257 violation.
 [flake8]