x/cloudbase-init
Code Issues Proposed changes

Add support for jinja as userdata template engine

Browse Source 
If the userdata starts with the text '## template:jinja',
Jinja2 templating engine will be used to render that
userdata.

This feature leverages the get_instance_data metadata
service implementation and tries to be as close as possible
with the cloud-init's implementation.

Change-Id: I79cb0066f7d65ae27867ac01b443e9be432b4dc3
This commit is contained in:
Adrian Vladu 2019-12-16 13:26:45 +02:00
parent c31c5b99f1
commit 493e57c195
12 changed files with 377 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
cloudbaseinit/plugins/common/userdata.py
View File

@ -26,6 +26,7 @@ from cloudbaseinit.plugins.common import execcmd
from cloudbaseinit.plugins.common.userdataplugins import factory
from cloudbaseinit.plugins.common import userdatautils
from cloudbaseinit.utils import encoding
from cloudbaseinit.utils.template_engine import factory as template_factory
from cloudbaseinit.utils import x509constants
@ -53,7 +54,7 @@ class UserDataPlugin(base.BasePlugin):
self._write_userdata(user_data, user_data_path)
if CONF.process_userdata:
return self._process_user_data(user_data)
return self._process_user_data(user_data, service)
return base.PLUGIN_EXECUTION_DONE, False
@staticmethod
@ -100,7 +101,7 @@ class UserDataPlugin(base.BasePlugin):
"The user data content is "
"either invalid or empty.")
def _process_user_data(self, user_data):
def _process_user_data(self, user_data, service):
plugin_status = base.PLUGIN_EXECUTION_DONE
reboot = False
headers = self._get_headers(user_data)
@ -122,7 +123,7 @@ class UserDataPlugin(base.BasePlugin):
return plugin_status, reboot
else:
return self._process_non_multi_part(user_data)
return self._process_non_multi_part(user_data, service)
def _process_part(self, part, user_data_plugins, user_handlers):
ret_val = None
@ -186,8 +187,16 @@ class UserDataPlugin(base.BasePlugin):
LOG.debug("Calling part handler \"__end__\" event")
handler_func(None, "__end__", None, None)
def _process_non_multi_part(self, user_data):
def _process_non_multi_part(self, user_data, service):
ret_val = None
template_engine = template_factory.get_template_engine(user_data)
if template_engine:
user_data = template_engine.render(
service.get_instance_data(),
user_data
)
if user_data.startswith(b'#cloud-config'):
user_data_plugins = factory.load_plugins()
cloud_config_plugin = user_data_plugins.get('text/cloud-config')

56
cloudbaseinit/tests/plugins/common/test_userdata.py
View File

@ -162,8 +162,10 @@ class UserDataPluginTest(unittest.TestCase):
mock_part = mock.MagicMock()
mock_parse_mime.return_value = [mock_part]
mock_process_part.return_value = (base.PLUGIN_EXECUTION_DONE, reboot)
mock_service = mock.MagicMock()
response = self._userdata._process_user_data(user_data=user_data)
response = self._userdata._process_user_data(user_data=user_data,
service=mock_service)
if user_data.startswith(b'Content-Type: multipart'):
mock_load_plugins.assert_called_once_with()
@ -172,7 +174,8 @@ class UserDataPluginTest(unittest.TestCase):
mock_load_plugins(), {})
self.assertEqual((base.PLUGIN_EXECUTION_DONE, reboot), response)
else:
mock_process_non_multi_part.assert_called_once_with(user_data)
mock_process_non_multi_part.assert_called_once_with(user_data,
mock_service)
self.assertEqual(mock_process_non_multi_part.return_value,
response)
@ -313,8 +316,9 @@ class UserDataPluginTest(unittest.TestCase):
'.execute_user_data_script')
def test_process_non_multi_part(self, mock_execute_user_data_script):
user_data = b'fake'
service = mock.MagicMock()
status, reboot = self._userdata._process_non_multi_part(
user_data=user_data)
user_data=user_data, service=service)
mock_execute_user_data_script.assert_called_once_with(user_data)
self.assertEqual(status, 1)
self.assertFalse(reboot)
@ -329,10 +333,11 @@ class UserDataPluginTest(unittest.TestCase):
b2NhbGhvc3QwHhcNMTUwNjE1MTAyODUxWhcNMjUwNjEyMTAyODUxWjAbMRkwFwYD
-----END CERTIFICATE-----
''').encode()
service = mock.MagicMock()
with testutils.LogSnatcher('cloudbaseinit.plugins.'
'common.userdata') as snatcher:
status, reboot = self._userdata._process_non_multi_part(
user_data=user_data)
user_data=user_data, service=service)
expected_logging = ['Found X509 certificate in userdata']
self.assertFalse(mock_execute_user_data_script.called)
@ -340,25 +345,54 @@ class UserDataPluginTest(unittest.TestCase):
self.assertEqual(1, status)
self.assertFalse(reboot)
@mock.patch('cloudbaseinit.utils.template_engine.factory.'
'get_template_engine')
@mock.patch('cloudbaseinit.plugins.common.userdataplugins.factory.'
'load_plugins')
def test_process_non_multi_part_cloud_config(self, mock_load_plugins):
user_data = b'#cloud-config'
def _test_process_non_multi_part_cloud_config(self, mock_load_plugins,
mock_load_templates,
user_data,
expected_userdata,
template_renderer=None):
mock_service = mock.MagicMock()
mock_return_value = mock.sentinel.return_value
mock_cloud_config_plugin = mock.Mock()
mock_cloud_config_plugin.process.return_value = mock_return_value
mock_load_plugins.return_value = {
'text/cloud-config': mock_cloud_config_plugin}
mock_load_templates.return_value = template_renderer
status, reboot = self._userdata._process_non_multi_part(
user_data=user_data)
user_data=user_data, service=mock_service)
if template_renderer:
mock_load_plugins.assert_called_once_with()
(mock_cloud_config_plugin
.process_non_multipart
.assert_called_once_with(expected_userdata))
mock_load_plugins.assert_called_once_with()
(mock_cloud_config_plugin
.process_non_multipart
.assert_called_once_with(user_data))
self.assertEqual(status, 1)
self.assertFalse(reboot)
def test_process_non_multi_part_cloud_config(self):
user_data = b'#cloud-config'
self._test_process_non_multi_part_cloud_config(
user_data=user_data, expected_userdata=user_data)
def test_process_non_multi_part_cloud_config_jinja(self):
user_data = b'## template:jinja\n#cloud-config'
expected_userdata = b'#cloud-config'
mock_template_renderer = mock.MagicMock()
mock_template_renderer.render.return_value = expected_userdata
self._test_process_non_multi_part_cloud_config(
user_data=user_data, expected_userdata=expected_userdata,
template_renderer=mock_template_renderer)
def test_process_non_multi_part_no_valid_template(self):
user_data = b'## template:none'
self._test_process_non_multi_part_cloud_config(
user_data=user_data, expected_userdata=user_data)
class TestCloudConfig(unittest.TestCase):
@classmethod

0
cloudbaseinit/tests/utils/template_engine/__init__.py Normal file
View File

30
cloudbaseinit/tests/utils/template_engine/test_base_template.py Normal file
View File

@ -0,0 +1,30 @@
# Copyright 2019 Cloudbase Solutions Srl
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import ddt
import unittest
from cloudbaseinit.utils.template_engine import base_template as bt
@ddt.ddt
class TestBaseTemplateEngine(unittest.TestCase):
@ddt.data((b'', b''),
(b'## template:jinja test', b''),
(b'## template:jinja \ntest', b'test'))
@ddt.unpack
def test_remove_template_definition(self, template, expected_output):
output = bt.BaseTemplateEngine.remove_template_definition(template)
self.assertEqual(expected_output, output)

58
cloudbaseinit/tests/utils/template_engine/test_factory.py Normal file
View File

@ -0,0 +1,58 @@
# Copyright 2019 Cloudbase Solutions Srl
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
try:
import unittest.mock as mock
except ImportError:
import mock
import unittest
from cloudbaseinit.utils.template_engine import factory
class FakeLoaderError(Exception):
pass
class TestTemplateFactory(unittest.TestCase):
def test_get_template_engine_empty(self):
fake_userdata = b''
result = factory.get_template_engine(fake_userdata)
self.assertEqual(result, None)
def test_get_template_engine_no_match(self):
fake_userdata = b'no match'
result = factory.get_template_engine(fake_userdata)
self.assertEqual(result, None)
def test_get_template_engine_not_supported(self):
fake_userdata = b'## template:fake'
result = factory.get_template_engine(fake_userdata)
self.assertEqual(result, None)
@mock.patch('cloudbaseinit.utils.classloader.ClassLoader')
def test_get_template_engine(self, mock_class_loader):
fake_userdata = b'## template:jinja'
mock_load_class = mock_class_loader.return_value.load_class
self.assertEqual(mock_load_class.return_value.return_value,
factory.get_template_engine(fake_userdata))
@mock.patch('cloudbaseinit.utils.classloader.ClassLoader')
def test_get_template_engine_class_not_found(self, mock_class_loader):
fake_userdata = b'## template:jinja'
mock_class_loader.return_value.load_class.side_effect = (
FakeLoaderError)
self.assertRaises(FakeLoaderError,
factory.get_template_engine, fake_userdata)

82
cloudbaseinit/tests/utils/template_engine/test_jinja2_template.py Normal file
View File

@ -0,0 +1,82 @@
# Copyright 2019 Cloudbase Solutions Srl
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import unittest
try:
import unittest.mock as mock
except ImportError:
import mock
from cloudbaseinit.utils.template_engine.jinja2_template import (
Jinja2TemplateEngine)
class TestJinja2TemplateEngine(unittest.TestCase):
@mock.patch('cloudbaseinit.utils.template_engine.base_template'
'.BaseTemplateEngine.remove_template_definition')
def _test_jinja_render_template(self, mock_remove_header,
fake_instance_data, expected_result,
fake_template = b'{{v1.local_hostname}}'):
mock_remove_header.return_value = fake_template
output = Jinja2TemplateEngine().render(fake_instance_data,
fake_template)
self.assertEqual(expected_result, output)
def test_jinja_render_template(self):
fake_instance_data = {
'v1': {
'local_hostname': 'fake_hostname'
}
}
expected_result = b'fake_hostname'
self._test_jinja_render_template(
fake_instance_data=fake_instance_data,
expected_result=expected_result)
def test_jinja_render_template_missing_variable(self):
fake_instance_data = {
'v1': {
'localhostname': 'fake_hostname'
}
}
expected_result = b'CI_MISSING_JINJA_VAR/local_hostname'
self._test_jinja_render_template(
fake_instance_data=fake_instance_data,
expected_result=expected_result)
def test_jinja_render_template_multiple_variables(self):
fake_instance_data = {
'v1': {
'localhostname': 'fake_hostname'
},
'ds': {
'meta_data': {
'hostname': 'fake_hostname'
},
'meta-data': {
'hostname': 'fake_hostname'
}
}
}
fake_template = b'{{ds.meta_data.hostname}}'
expected_result = b'fake_hostname'
self._test_jinja_render_template(
fake_instance_data=fake_instance_data,
expected_result=expected_result,
fake_template=fake_template)

0
cloudbaseinit/utils/template_engine/__init__.py Normal file
View File

60
cloudbaseinit/utils/template_engine/base_template.py Normal file
View File

@ -0,0 +1,60 @@
# Copyright 2019 Cloudbase Solutions Srl
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import abc
import re
import six
@six.add_metaclass(abc.ABCMeta)
class BaseTemplateEngine(object):
def __init__(self):
self._template_matcher = re.compile(r"##\s*template:(.*)", re.I)
@abc.abstractmethod
def get_template_type(self):
"""Return the template type for the class loader"""
pass
@abc.abstractmethod
def render(self, data, template):
"""Renders the template according to the data dictionary
The data variable is a dict which contains the key-values
that will be used to render the template.
The template is an encoded string which can contain special
constructions that will be used by the template engine.
The return value will be an encoded string.
"""
def load(self, data):
"""Returns True if the template header matches, False otherwise"""
template_type_matcher = self._template_matcher.match(data.decode())
template_type = template_type_matcher.group(1).lower().strip()
if self.get_template_type() == template_type:
return True
@staticmethod
def remove_template_definition(raw_template):
# Remove the first line, as it contains the template definition
template_split = raw_template.split(b"\n", 1)
if len(template_split) == 2:
# return the template without the header
return template_split[1]
# the template has just one line, return empty encoded string
return b''

37
cloudbaseinit/utils/template_engine/factory.py Normal file
View File

@ -0,0 +1,37 @@
# Copyright 2019 Cloudbase Solutions Srl
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from cloudbaseinit.utils import classloader
from oslo_log import log as oslo_logging
TEMPLATE_ENGINE_CLASS_PATHS = ["cloudbaseinit.utils.template_engine"
".jinja2_template.Jinja2TemplateEngine"]
LOG = oslo_logging.getLogger(__name__)
def get_template_engine(user_data):
"""Returns the first template engine that loads correctly"""
cl = classloader.ClassLoader()
for class_path in TEMPLATE_ENGINE_CLASS_PATHS:
tpl_engine = cl.load_class(class_path)()
try:
if tpl_engine.load(user_data):
LOG.info("Using template engine: %s"
% tpl_engine.get_template_type())
return tpl_engine
except Exception as ex:
LOG.error("Failed to load template engine '%s'" % class_path)
LOG.exception(ex)
return

50
cloudbaseinit/utils/template_engine/jinja2_template.py Normal file
View File

@ -0,0 +1,50 @@
# Copyright 2019 Cloudbase Solutions Srl
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import jinja2
from cloudbaseinit.utils.template_engine import base_template
from jinja2 import runtime
MISSING_JINJA_VARIABLE = u'CI_MISSING_JINJA_VAR/'
@runtime.implements_to_string
class MissingJinjaVariable(jinja2.DebugUndefined):
"""Missing Jinja2 variable class."""
def __str__(self):
return u'%s%s' % (MISSING_JINJA_VARIABLE, self._undefined_name)
class Jinja2TemplateEngine(base_template.BaseTemplateEngine):
def get_template_type(self):
return 'jinja'
def render(self, data, raw_template):
"""Renders the template using Jinja2 template engine
The data variable is a dict which contains the key-values
that will be used to render the template.
The template is an encoded string which can contain special
constructions that will be used by the template engine.
The return value will be an encoded string.
"""
template = self.remove_template_definition(raw_template).decode()
jinja_template = jinja2.Template(template,
trim_blocks=True,
undefined=MissingJinjaVariable,)
return jinja_template.render(**data).encode()

1
requirements.txt
View File

@ -12,6 +12,7 @@ netifaces
PyYAML
requests
untangle==1.1.1
jinja2
pywin32;sys_platform=="win32"
comtypes;sys_platform=="win32"
pymi;sys_platform=="win32"

1
test-requirements.txt
View File

@ -10,3 +10,4 @@ stestr>=2.0.0
openstackdocstheme>=1.11.0 # Apache-2.0
# releasenotes
reno>=1.8.0 # Apache-2.0
ddt