diff --git a/mistral/config.py b/mistral/config.py index 67464f303..3c8486409 100644 --- a/mistral/config.py +++ b/mistral/config.py @@ -449,6 +449,59 @@ os_actions_mapping_path = cfg.StrOpt( 'directory or absolute.' ) +yaql_opts = [ + cfg.IntOpt( + 'limit_iterators', + default=-1, + min=-1, + help=_('Limit iterators by the given number of elements. When set, ' + 'each time any function declares its parameter to be iterator, ' + 'that iterator is modified to not produce more than a given ' + 'number of items. If not set (or set to -1) the result data is ' + 'allowed to contain endless iterators that would cause errors ' + 'if the result where to be serialized.') + ), + cfg.IntOpt( + 'memory_quota', + default=-1, + min=-1, + help=_('The memory usage quota (in bytes) for all data produced by ' + 'the expression (or any part of it). -1 means no limitation.') + ), + cfg.BoolOpt( + 'convert_tuples_to_lists', + default=True, + help=_('When set to true, yaql converts all tuples in the expression ' + 'result to lists.') + ), + cfg.BoolOpt( + 'convert_sets_to_lists', + default=False, + help=_('When set to true, yaql converts all sets in the expression ' + 'result to lists. Otherwise the produced result may contain ' + 'sets that are not JSON-serializable.') + ), + cfg.BoolOpt( + 'iterable_dicts', + default=False, + help=_('When set to true, dictionaries are considered to be iterable ' + 'and iteration over dictionaries produces their keys (as in ' + 'Python and yaql 0.2).') + ), + cfg.StrOpt( + 'keyword_operator', + default='=>', + help=_('Allows one to configure keyword/mapping symbol. ' + 'Ability to pass named arguments can be disabled altogether ' + 'if empty string is provided.') + ), + cfg.BoolOpt( + 'allow_delegates', + default=False, + help=_('Enables or disables delegate expression parsing.') + ) +] + CONF = cfg.CONF API_GROUP = 'api' @@ -464,6 +517,7 @@ EXECUTION_EXPIRATION_POLICY_GROUP = 'execution_expiration_policy' PROFILER_GROUP = profiler.list_opts()[0][0] KEYCLOAK_OIDC_GROUP = "keycloak_oidc" OPENSTACK_ACTIONS_GROUP = 'openstack_actions' +YAQL_GROUP = "yaql" CONF.register_opt(wf_trace_log_name_opt) @@ -489,6 +543,7 @@ CONF.register_opts(coordination_opts, group=COORDINATION_GROUP) CONF.register_opts(profiler_opts, group=PROFILER_GROUP) CONF.register_opts(keycloak_oidc_opts, group=KEYCLOAK_OIDC_GROUP) CONF.register_opts(openstack_actions_opts, group=OPENSTACK_ACTIONS_GROUP) +CONF.register_opts(yaql_opts, group=YAQL_GROUP) CLI_OPTS = [ use_debugger_opt, @@ -535,6 +590,7 @@ def list_opts(): (PROFILER_GROUP, profiler_opts), (KEYCLOAK_OIDC_GROUP, keycloak_oidc_opts), (OPENSTACK_ACTIONS_GROUP, openstack_actions_opts), + (YAQL_GROUP, yaql_opts), (None, default_group_opts) ] diff --git a/mistral/expressions/yaql_expression.py b/mistral/expressions/yaql_expression.py index 8cd500a3d..e0bc534af 100644 --- a/mistral/expressions/yaql_expression.py +++ b/mistral/expressions/yaql_expression.py @@ -23,12 +23,52 @@ import six from yaql.language import exceptions as yaql_exc from yaql.language import factory +from mistral.config import cfg from mistral import exceptions as exc from mistral.expressions.base_expression import Evaluator +from mistral import utils from mistral.utils import expression_utils LOG = logging.getLogger(__name__) -YAQL_ENGINE = factory.YaqlFactory().create() + +_YAQL_CONF = cfg.CONF.yaql + + +def get_yaql_engine_options(): + return { + "yaql.limitIterators": _YAQL_CONF.limit_iterators, + "yaql.memoryQuota": _YAQL_CONF.memory_quota, + "yaql.convertTuplesToLists": _YAQL_CONF.convert_tuples_to_lists, + "yaql.convertSetsToLists": _YAQL_CONF.convert_sets_to_lists, + "yaql.iterableDicts": _YAQL_CONF.iterable_dicts, + "yaql.convertOutputData": True + } + + +def create_yaql_engine_class(keyword_operator, allow_delegates, + engine_options): + return factory.YaqlFactory( + keyword_operator=keyword_operator, + allow_delegates=allow_delegates + ).create(options=engine_options) + + +YAQL_ENGINE = create_yaql_engine_class( + _YAQL_CONF.keyword_operator, + _YAQL_CONF.allow_delegates, + get_yaql_engine_options() +) + +LOG.info( + "YAQL engine has been initialized with the options: \n%s", + utils.merge_dicts( + get_yaql_engine_options(), + { + "keyword_operator": _YAQL_CONF.keyword_operator, + "allow_delegates": _YAQL_CONF.allow_delegates + } + ) +) INLINE_YAQL_REGEXP = '<%.*?%>' diff --git a/mistral/tests/unit/engine/test_yaql_functions.py b/mistral/tests/unit/engine/test_yaql_functions.py index dee0b5599..c3a6c6828 100644 --- a/mistral/tests/unit/engine/test_yaql_functions.py +++ b/mistral/tests/unit/engine/test_yaql_functions.py @@ -389,3 +389,45 @@ class YAQLFunctionsEngineTest(engine_test_base.EngineTestCase): wf_ex.created_at.isoformat(' '), execution['created_at'] ) + + def test_yaml_dump_function(self): + wf_text = """--- + version: '2.0' + + wf: + tasks: + task1: + publish: + data: <% {key1 => foo, key2 => bar} %> + on-success: task2 + + task2: + publish: + yaml_str: <% yaml_dump($.data) %> + json_str: <% json_dump($.data) %> + """ + + wf_service.create_workflows(wf_text) + + wf_ex = self.engine.start_workflow('wf') + + self.await_workflow_success(wf_ex.id) + + with db_api.transaction(read_only=True): + wf_ex = db_api.get_workflow_execution(wf_ex.id) + + task_ex = self._assert_single_item( + wf_ex.task_executions, + name='task2' + ) + + yaml_str = task_ex.published['yaml_str'] + json_str = task_ex.published['json_str'] + + self.assertIsNotNone(yaml_str) + self.assertIn('key1: foo', yaml_str) + self.assertIn('key2: bar', yaml_str) + + self.assertIsNotNone(json_str) + self.assertIn('"key1": "foo"', json_str) + self.assertIn('"key2": "bar"', json_str) diff --git a/mistral/tests/unit/expressions/test_yaql_expression.py b/mistral/tests/unit/expressions/test_yaql_expression.py index 878565064..188e60fdb 100644 --- a/mistral/tests/unit/expressions/test_yaql_expression.py +++ b/mistral/tests/unit/expressions/test_yaql_expression.py @@ -21,11 +21,14 @@ import warnings import mock +from mistral.config import cfg from mistral import exceptions as exc from mistral.expressions import yaql_expression as expr from mistral.tests.unit import base from mistral import utils +CONF = cfg.CONF + DATA = { "server": { "id": "03ea824a-aa24-4105-9131-66c48ae54acf", @@ -297,3 +300,40 @@ class InlineYAQLEvaluatorTest(base.BaseTest): self.assertRaises(exc.YaqlEvaluationException, self._evaluator.validate, {'a': 1}) + + def test_set_of_dicts(self): + self.override_config('convert_sets_to_lists', True, 'yaql') + + def _restore_engine(old_engine): + expr.YAQL_ENGINE = old_engine + + self.addCleanup(_restore_engine, expr.YAQL_ENGINE) + + expr.YAQL_ENGINE = expr.create_yaql_engine_class( + CONF.yaql.keyword_operator, + CONF.yaql.allow_delegates, + expr.get_yaql_engine_options() + ) + + my_list = [ + { + "k1": "v1", + "k2": "v2" + }, + { + "k11": "v11", + "k12": "v12" + } + ] + + res = self._evaluator.evaluate( + '<% $.my_list.toSet() %>', + {"my_list": my_list} + ) + + self.assertIsInstance(res, list) + self.assertEqual(2, len(res)) + + # The order may be different so we can't use "assertListEqual". + self.assertTrue(my_list[0] == res[0] or my_list[1] == res[0]) + self.assertTrue(my_list[0] == res[1] or my_list[1] == res[1]) diff --git a/mistral/utils/expression_utils.py b/mistral/utils/expression_utils.py index 723587a67..1155b10f0 100644 --- a/mistral/utils/expression_utils.py +++ b/mistral/utils/expression_utils.py @@ -20,11 +20,20 @@ from oslo_log import log as logging from oslo_serialization import jsonutils from stevedore import extension import yaml +from yaml import representer import yaql +from yaql.language import utils as yaql_utils + from mistral.db.v2 import api as db_api from mistral import utils +# TODO(rakhmerov): it's work around the bug in YAQL. +# YAQL shouldn't expose internal types to custom functions. +representer.SafeRepresenter.add_representer( + yaql_utils.FrozenDict, + representer.SafeRepresenter.represent_dict +) LOG = logging.getLogger(__name__) ROOT_YAQL_CONTEXT = None @@ -39,7 +48,7 @@ def get_yaql_context(data_context): _register_yaql_functions(ROOT_YAQL_CONTEXT) new_ctx = ROOT_YAQL_CONTEXT.create_child_context() - new_ctx['$'] = data_context + new_ctx['$'] = yaql_utils.convert_input_data(data_context) if isinstance(data_context, dict): new_ctx['__env'] = data_context.get('__env')