openstack/mistral
Code Issues Proposed changes

Add YAQL engine options

Browse Source 
Change-Id: I2e2c85c8be90ee62c8f37f002e21098f17ba6f5c
Closes-Bug: #1700572
Closes-Bug: #1772864
This commit is contained in:
Xavier Hardy 2017-06-27 10:23:01 +02:00 committed by Renat Akhmerov
parent 573d2d0682
commit fe0d441082
5 changed files with 189 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
mistral/config.py
View File

@ -449,6 +449,59 @@ os_actions_mapping_path = cfg.StrOpt(
'directory or absolute.'
)
yaql_opts = [
cfg.IntOpt(
'limit_iterators',
default=-1,
min=-1,
help=_('Limit iterators by the given number of elements. When set, '
'each time any function declares its parameter to be iterator, '
'that iterator is modified to not produce more than a given '
'number of items. If not set (or set to -1) the result data is '
'allowed to contain endless iterators that would cause errors '
'if the result where to be serialized.')
),
cfg.IntOpt(
'memory_quota',
default=-1,
min=-1,
help=_('The memory usage quota (in bytes) for all data produced by '
'the expression (or any part of it). -1 means no limitation.')
),
cfg.BoolOpt(
'convert_tuples_to_lists',
default=True,
help=_('When set to true, yaql converts all tuples in the expression '
'result to lists.')
),
cfg.BoolOpt(
'convert_sets_to_lists',
default=False,
help=_('When set to true, yaql converts all sets in the expression '
'result to lists. Otherwise the produced result may contain '
'sets that are not JSON-serializable.')
),
cfg.BoolOpt(
'iterable_dicts',
default=False,
help=_('When set to true, dictionaries are considered to be iterable '
'and iteration over dictionaries produces their keys (as in '
'Python and yaql 0.2).')
),
cfg.StrOpt(
'keyword_operator',
default='=>',
help=_('Allows one to configure keyword/mapping symbol. '
'Ability to pass named arguments can be disabled altogether '
'if empty string is provided.')
),
cfg.BoolOpt(
'allow_delegates',
default=False,
help=_('Enables or disables delegate expression parsing.')
)
]
CONF = cfg.CONF
API_GROUP = 'api'
@ -464,6 +517,7 @@ EXECUTION_EXPIRATION_POLICY_GROUP = 'execution_expiration_policy'
PROFILER_GROUP = profiler.list_opts()[0][0]
KEYCLOAK_OIDC_GROUP = "keycloak_oidc"
OPENSTACK_ACTIONS_GROUP = 'openstack_actions'
YAQL_GROUP = "yaql"
CONF.register_opt(wf_trace_log_name_opt)
@ -489,6 +543,7 @@ CONF.register_opts(coordination_opts, group=COORDINATION_GROUP)
CONF.register_opts(profiler_opts, group=PROFILER_GROUP)
CONF.register_opts(keycloak_oidc_opts, group=KEYCLOAK_OIDC_GROUP)
CONF.register_opts(openstack_actions_opts, group=OPENSTACK_ACTIONS_GROUP)
CONF.register_opts(yaql_opts, group=YAQL_GROUP)
CLI_OPTS = [
use_debugger_opt,
@ -535,6 +590,7 @@ def list_opts():
(PROFILER_GROUP, profiler_opts),
(KEYCLOAK_OIDC_GROUP, keycloak_oidc_opts),
(OPENSTACK_ACTIONS_GROUP, openstack_actions_opts),
(YAQL_GROUP, yaql_opts),
(None, default_group_opts)
]

42
mistral/expressions/yaql_expression.py
View File

@ -23,12 +23,52 @@ import six
from yaql.language import exceptions as yaql_exc
from yaql.language import factory
from mistral.config import cfg
from mistral import exceptions as exc
from mistral.expressions.base_expression import Evaluator
from mistral import utils
from mistral.utils import expression_utils
LOG = logging.getLogger(__name__)
YAQL_ENGINE = factory.YaqlFactory().create()
_YAQL_CONF = cfg.CONF.yaql
def get_yaql_engine_options():
return {
"yaql.limitIterators": _YAQL_CONF.limit_iterators,
"yaql.memoryQuota": _YAQL_CONF.memory_quota,
"yaql.convertTuplesToLists": _YAQL_CONF.convert_tuples_to_lists,
"yaql.convertSetsToLists": _YAQL_CONF.convert_sets_to_lists,
"yaql.iterableDicts": _YAQL_CONF.iterable_dicts,
"yaql.convertOutputData": True
}
def create_yaql_engine_class(keyword_operator, allow_delegates,
engine_options):
return factory.YaqlFactory(
keyword_operator=keyword_operator,
allow_delegates=allow_delegates
).create(options=engine_options)
YAQL_ENGINE = create_yaql_engine_class(
_YAQL_CONF.keyword_operator,
_YAQL_CONF.allow_delegates,
get_yaql_engine_options()
)
LOG.info(
"YAQL engine has been initialized with the options: \n%s",
utils.merge_dicts(
get_yaql_engine_options(),
{
"keyword_operator": _YAQL_CONF.keyword_operator,
"allow_delegates": _YAQL_CONF.allow_delegates
}
)
)
INLINE_YAQL_REGEXP = '<%.*?%>'

42
mistral/tests/unit/engine/test_yaql_functions.py
View File

@ -389,3 +389,45 @@ class YAQLFunctionsEngineTest(engine_test_base.EngineTestCase):
wf_ex.created_at.isoformat(' '),
execution['created_at']
)
def test_yaml_dump_function(self):
wf_text = """---
version: '2.0'
wf:
tasks:
task1:
publish:
data: <% {key1 => foo, key2 => bar} %>
on-success: task2
task2:
publish:
yaml_str: <% yaml_dump($.data) %>
json_str: <% json_dump($.data) %>
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_success(wf_ex.id)
with db_api.transaction(read_only=True):
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = self._assert_single_item(
wf_ex.task_executions,
name='task2'
)
yaml_str = task_ex.published['yaml_str']
json_str = task_ex.published['json_str']
self.assertIsNotNone(yaml_str)
self.assertIn('key1: foo', yaml_str)
self.assertIn('key2: bar', yaml_str)
self.assertIsNotNone(json_str)
self.assertIn('"key1": "foo"', json_str)
self.assertIn('"key2": "bar"', json_str)

40
mistral/tests/unit/expressions/test_yaql_expression.py
View File

@ -21,11 +21,14 @@ import warnings
import mock
from mistral.config import cfg
from mistral import exceptions as exc
from mistral.expressions import yaql_expression as expr
from mistral.tests.unit import base
from mistral import utils
CONF = cfg.CONF
DATA = {
"server": {
"id": "03ea824a-aa24-4105-9131-66c48ae54acf",
@ -297,3 +300,40 @@ class InlineYAQLEvaluatorTest(base.BaseTest):
self.assertRaises(exc.YaqlEvaluationException,
self._evaluator.validate,
{'a': 1})
def test_set_of_dicts(self):
self.override_config('convert_sets_to_lists', True, 'yaql')
def _restore_engine(old_engine):
expr.YAQL_ENGINE = old_engine
self.addCleanup(_restore_engine, expr.YAQL_ENGINE)
expr.YAQL_ENGINE = expr.create_yaql_engine_class(
CONF.yaql.keyword_operator,
CONF.yaql.allow_delegates,
expr.get_yaql_engine_options()
)
my_list = [
{
"k1": "v1",
"k2": "v2"
},
{
"k11": "v11",
"k12": "v12"
}
]
res = self._evaluator.evaluate(
'<% $.my_list.toSet() %>',
{"my_list": my_list}
)
self.assertIsInstance(res, list)
self.assertEqual(2, len(res))
# The order may be different so we can't use "assertListEqual".
self.assertTrue(my_list[0] == res[0] or my_list[1] == res[0])
self.assertTrue(my_list[0] == res[1] or my_list[1] == res[1])

11
mistral/utils/expression_utils.py
View File

@ -20,11 +20,20 @@ from oslo_log import log as logging
from oslo_serialization import jsonutils
from stevedore import extension
import yaml
from yaml import representer
import yaql
from yaql.language import utils as yaql_utils
from mistral.db.v2 import api as db_api
from mistral import utils
# TODO(rakhmerov): it's work around the bug in YAQL.
# YAQL shouldn't expose internal types to custom functions.
representer.SafeRepresenter.add_representer(
yaql_utils.FrozenDict,
representer.SafeRepresenter.represent_dict
)
LOG = logging.getLogger(__name__)
ROOT_YAQL_CONTEXT = None
@ -39,7 +48,7 @@ def get_yaql_context(data_context):
_register_yaql_functions(ROOT_YAQL_CONTEXT)
new_ctx = ROOT_YAQL_CONTEXT.create_child_context()
new_ctx['$'] = data_context
new_ctx['$'] = yaql_utils.convert_input_data(data_context)
if isinstance(data_context, dict):
new_ctx['__env'] = data_context.get('__env')