From 29105932c07de79aed7b0fc68a611cc6be60194f Mon Sep 17 00:00:00 2001 From: Adrian Vladu Date: Fri, 6 Dec 2019 17:04:28 +0200 Subject: [PATCH] Add Google Cloud Engine metadata service Add cloudbaseinit.metadata.services.gceservice.GCEService that supports Google Cloud Engine. Supported features for the metadata service: * instance id * hostname * userdata * ssh keys Change-Id: I9e5e2cbcaa0953fc0c0ae8117e258713ac6443b7 --- cloudbaseinit/conf/factory.py | 1 + cloudbaseinit/conf/gce.py | 51 +++++ cloudbaseinit/metadata/services/gceservice.py | 184 ++++++++++++++++++ .../metadata/services/test_gceservice.py | 172 ++++++++++++++++ doc/source/services.rst | 34 ++++ 5 files changed, 442 insertions(+) create mode 100644 cloudbaseinit/conf/gce.py create mode 100644 cloudbaseinit/metadata/services/gceservice.py create mode 100644 cloudbaseinit/tests/metadata/services/test_gceservice.py diff --git a/cloudbaseinit/conf/factory.py b/cloudbaseinit/conf/factory.py index ad2b37e5..665ce73d 100644 --- a/cloudbaseinit/conf/factory.py +++ b/cloudbaseinit/conf/factory.py @@ -25,6 +25,7 @@ _OPT_PATHS = ( 'cloudbaseinit.conf.ovf.OvfOptions', 'cloudbaseinit.conf.packet.PacketOptions', 'cloudbaseinit.conf.vmwareguestinfo.VMwareGuestInfoConfigOptions', + 'cloudbaseinit.conf.gce.GCEOptions', ) diff --git a/cloudbaseinit/conf/gce.py b/cloudbaseinit/conf/gce.py new file mode 100644 index 00000000..ea269deb --- /dev/null +++ b/cloudbaseinit/conf/gce.py @@ -0,0 +1,51 @@ +# Copyright 2020 Cloudbase Solutions Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Config options available for the GCE metadata service.""" + +from oslo_config import cfg + +from cloudbaseinit.conf import base as conf_base + + +class GCEOptions(conf_base.Options): + + """Config options available for the GCE metadata service.""" + + def __init__(self, config): + super(GCEOptions, self).__init__(config, group="gce") + self._options = [ + cfg.StrOpt( + "metadata_base_url", + default="http://metadata.google.internal/computeMetadata/v1/", + help="The base URL where the service looks for metadata"), + cfg.BoolOpt( + "https_allow_insecure", default=False, + help="Whether to disable the validation of HTTPS " + "certificates."), + cfg.StrOpt( + "https_ca_bundle", default=None, + help="The path to a CA_BUNDLE file or directory with " + "certificates of trusted CAs."), + ] + + def register(self): + """Register the current options to the global ConfigOpts object.""" + group = cfg.OptGroup(self.group_name, title='GCE Options') + self._config.register_group(group) + self._config.register_opts(self._options, group=group) + + def list(self): + """Return a list which contains all the available options.""" + return self._options diff --git a/cloudbaseinit/metadata/services/gceservice.py b/cloudbaseinit/metadata/services/gceservice.py new file mode 100644 index 00000000..c2cfb43a --- /dev/null +++ b/cloudbaseinit/metadata/services/gceservice.py @@ -0,0 +1,184 @@ +# Copyright 2020 Cloudbase Solutions Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import base64 +import json + +from datetime import datetime +from oslo_log import log as oslo_logging + +from cloudbaseinit import conf as cloudbaseinit_conf +from cloudbaseinit.metadata.services import base + +CONF = cloudbaseinit_conf.CONF +LOG = oslo_logging.getLogger(__name__) + +GCE_METADATA_HEADERS = {'Metadata-Flavor': 'Google'} +MD_INSTANCE = "instance" +MD_INSTANCE_ATTR = "%s/attributes" % MD_INSTANCE +MD_PROJECT_ATTR = "project/attributes" + + +class GCEService(base.BaseHTTPMetadataService): + + def __init__(self): + super(GCEService, self).__init__( + base_url=CONF.gce.metadata_base_url, + https_allow_insecure=CONF.gce.https_allow_insecure, + https_ca_bundle=CONF.gce.https_ca_bundle) + self._enable_retry = True + + def _http_request(self, url, data=None, headers=None, method=None): + headers = headers or {} + headers.update(GCE_METADATA_HEADERS) + + return super(GCEService, self)._http_request(url, data, + headers, method) + + def load(self): + super(GCEService, self).load() + + try: + self.get_host_name() + return True + except base.NotExistingMetadataException: + LOG.debug("Metadata not found at URL '%s'", + CONF.gce.metadata_base_url) + + def get_host_name(self): + return self._get_cache_data('%s/name' % MD_INSTANCE, decode=True) + + def get_instance_id(self): + return self._get_cache_data('%s/id' % MD_INSTANCE, decode=True) + + def get_user_data(self): + user_data = self._get_cache_data('%s/user-data' % MD_INSTANCE_ATTR) + try: + encoding = self._get_cache_data( + '%s/user-data-encoding' % MD_INSTANCE_ATTR, + decode=True) + if encoding: + if encoding == 'base64': + user_data = base64.b64decode(user_data) + else: + LOG.warning("Encoding '%s' not supported. " + "Falling back to plaintext", encoding) + except base.NotExistingMetadataException: + LOG.info('Userdata encoding could not be found in the metadata.') + + return user_data + + def _is_ssh_key_valid(self, expire_on): + if not expire_on: + return True + try: + time_format = '%Y-%m-%dT%H:%M:%S+0000' + expire_time = datetime.strptime(expire_on, time_format) + return datetime.utcnow() <= expire_time + except ValueError: + # Note(ader1990): Return True to be consistent with cloud-init + return True + + def _parse_gce_ssh_key(self, raw_ssh_key): + # GCE public keys have a special format defined here: + # https://cloud.google.com/compute/docs/instances/adding-removing-ssh-keys#sshkeyformat + INVALID_SSH_KEY_MSG = "Skipping invalid SSH key %s" + header_username = None + meta_username = None + + if not raw_ssh_key: + return + + ssh_key = raw_ssh_key.strip() + + # Key is in the format: USERNAME:ssh-rsa ... + # Remove the username from the key + ssh_key_split = ssh_key.split(':', 1) + if len(ssh_key_split) != 2: + LOG.warning(INVALID_SSH_KEY_MSG, ssh_key) + return + + header_username, ssh_key = ssh_key_split + + key_parts = ssh_key.split(' ') + len_key_parts = len(key_parts) + + if len_key_parts < 3: + # Key format not supported: USERNAME:ssh-rsa [KEY] + LOG.warning(INVALID_SSH_KEY_MSG, ssh_key) + return + elif len_key_parts == 3: + # Key format: USERNAME:ssh-rsa [KEY] [USERNAME] + meta_username = key_parts[2] + else: + # Key format: USERNAME:ssh-rsa [KEY] google-ssh [JSON_METADATA] + delimiter = 'google-ssh' + json_key_parts = ssh_key.split(delimiter) + if (len(json_key_parts) == 2 and json_key_parts[1]): + ssh_key_metadata = json.loads(json_key_parts[1].strip()) + meta_username = ssh_key_metadata['userName'] + if not self._is_ssh_key_valid(ssh_key_metadata['expireOn']): + LOG.warning("Skipping expired key: %s", ssh_key) + return + ssh_key = '%s %s' % (json_key_parts[0].strip(), meta_username) + else: + LOG.warning(INVALID_SSH_KEY_MSG, ssh_key) + return + + if not (header_username == meta_username == CONF.username): + LOG.warning("Skipping key due to non matching username: %s", + ssh_key) + return + + return ssh_key + + def _get_ssh_keys(self, locations): + ssh_keys = [] + for location in locations: + try: + raw_ssh_keys = self._get_cache_data(location, decode=True) + ssh_keys += raw_ssh_keys.strip().splitlines() + except base.NotExistingMetadataException: + LOG.warning("SSH keys not found at location %s", location) + + return ssh_keys + + def get_public_keys(self): + ssh_keys = [] + raw_ssh_keys = [] + block_project_keys = False + key_locations = ["%s/ssh-keys" % MD_INSTANCE_ATTR] + + # Use GCE latest metadata, where the SSH keys are found + # only at hyphenated locations + try: + block_key = "%s/block-project-ssh-keys" % MD_INSTANCE_ATTR + if self._get_cache_data(block_key, decode=True) == 'true': + block_project_keys = True + except base.NotExistingMetadataException: + LOG.debug('block-project-ssh-keys not present') + + if not block_project_keys: + key_locations += [ + "%s/ssh-keys" % MD_PROJECT_ATTR + ] + + raw_ssh_keys += self._get_ssh_keys(key_locations) + + for raw_ssh_key in raw_ssh_keys: + ssh_key = self._parse_gce_ssh_key(raw_ssh_key) + if ssh_key: + ssh_keys.append(ssh_key) + + return ssh_keys diff --git a/cloudbaseinit/tests/metadata/services/test_gceservice.py b/cloudbaseinit/tests/metadata/services/test_gceservice.py new file mode 100644 index 00000000..b2121757 --- /dev/null +++ b/cloudbaseinit/tests/metadata/services/test_gceservice.py @@ -0,0 +1,172 @@ +# Copyright 2020 Cloudbase Solutions Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import ddt +import importlib +import unittest + +try: + import unittest.mock as mock +except ImportError: + import mock + +from cloudbaseinit import conf as cloudbaseinit_conf +from cloudbaseinit.tests import testutils + + +CONF = cloudbaseinit_conf.CONF +BASE_MODULE_PATH = ("cloudbaseinit.metadata.services.base." + "BaseHTTPMetadataService") +MODULE_PATH = "cloudbaseinit.metadata.services.gceservice" + + +@ddt.ddt +class GCEServiceTest(unittest.TestCase): + + def setUp(self): + self._win32com_mock = mock.MagicMock() + self._ctypes_mock = mock.MagicMock() + self._ctypes_util_mock = mock.MagicMock() + self._win32com_client_mock = mock.MagicMock() + self._pywintypes_mock = mock.MagicMock() + self._module_patcher = mock.patch.dict( + 'sys.modules', + {'win32com': self._win32com_mock, + 'ctypes': self._ctypes_mock, + 'ctypes.util': self._ctypes_util_mock, + 'win32com.client': self._win32com_client_mock, + 'pywintypes': self._pywintypes_mock}) + self._module_patcher.start() + self.addCleanup(self._module_patcher.stop) + + self._module = importlib.import_module(MODULE_PATH) + self._service = self._module.GCEService() + self.snatcher = testutils.LogSnatcher(MODULE_PATH) + + @mock.patch(MODULE_PATH + ".GCEService._get_cache_data") + def test_get_host_name(self, mock_get_cache_data): + response = self._service.get_host_name() + mock_get_cache_data.assert_called_once_with( + 'instance/name', decode=True) + self.assertEqual(mock_get_cache_data.return_value, + response) + + @mock.patch(MODULE_PATH + ".GCEService._get_cache_data") + def test_get_instance_id(self, mock_get_cache_data): + response = self._service.get_instance_id() + mock_get_cache_data.assert_called_once_with( + 'instance/id', decode=True) + self.assertEqual(mock_get_cache_data.return_value, + response) + + @mock.patch(MODULE_PATH + ".GCEService._get_cache_data") + def test_get_user_data(self, mock_get_cache_data): + response = self._service.get_user_data() + userdata_key = "%s/user-data" % self._module.MD_INSTANCE_ATTR + userdata_enc_key = ( + "%s/user-data-encoding" % self._module.MD_INSTANCE_ATTR) + mock_calls = [mock.call(userdata_key), + mock.call(userdata_enc_key, decode=True)] + mock_get_cache_data.assert_has_calls(mock_calls) + self.assertEqual(mock_get_cache_data.return_value, + response) + + @mock.patch(MODULE_PATH + ".GCEService._get_cache_data") + def test_get_user_data_b64(self, mock_get_cache_data): + user_data = b'fake userdata' + user_data_b64 = 'ZmFrZSB1c2VyZGF0YQ==' + userdata_key = "%s/user-data" % self._module.MD_INSTANCE_ATTR + userdata_enc_key = ( + "%s/user-data-encoding" % self._module.MD_INSTANCE_ATTR) + + def _get_cache_data_side_effect(*args, **kwargs): + if args[0] == ("%s/user-data" % self._module.MD_INSTANCE_ATTR): + return user_data_b64 + return 'base64' + mock_get_cache_data.side_effect = _get_cache_data_side_effect + + response = self._service.get_user_data() + + mock_calls = [mock.call(userdata_key), + mock.call(userdata_enc_key, decode=True)] + mock_get_cache_data.assert_has_calls(mock_calls) + self.assertEqual(response, user_data) + + @mock.patch(MODULE_PATH + ".GCEService._get_cache_data") + @mock.patch(MODULE_PATH + ".GCEService._get_ssh_keys") + @mock.patch(MODULE_PATH + ".GCEService._parse_gce_ssh_key") + def _test_get_public_keys_block_project(self, mock_parse_keys, + mock_get_ssh_keys, + mock_cache_data, + cache_data_result=False): + expected_response = [] + + if cache_data_result: + second_call_get_ssh = [ + '%s/ssh-keys' % self._module.MD_INSTANCE_ATTR, + '%s/ssh-keys' % self._module.MD_PROJECT_ATTR] + mock_cache_data.return_value = 'false' + else: + second_call_get_ssh = [ + '%s/ssh-keys' % self._module.MD_INSTANCE_ATTR] + mock_cache_data.return_value = 'true' + mock_get_ssh_keys.return_value = [] + response = self._service.get_public_keys() + mock_calls = [mock.call(second_call_get_ssh)] + mock_get_ssh_keys.assert_has_calls(mock_calls) + + self.assertEqual(mock_parse_keys.call_count, 0) + self.assertEqual(response, expected_response) + + def test_get_public_keys_block_project_check(self): + self._test_get_public_keys_block_project(cache_data_result=False) + + def test_get_public_keys_block_project(self): + self._test_get_public_keys_block_project(cache_data_result=True) + + @mock.patch(MODULE_PATH + ".GCEService._get_cache_data") + def test__get_ssh_keys(self, mock_get_cache_data): + fake_key = 'fake key' + expected_response = [fake_key] * 3 + key_locations = ['location'] * 3 + mock_get_cache_data.return_value = fake_key + response = self._service._get_ssh_keys(key_locations) + self.assertEqual(response, expected_response) + + @ddt.data((None, True), + ('not a date', True), + ('2018-12-04T20:12:00+0000', False)) + @ddt.unpack + def test__is_ssh_key_valid(self, expire_on, expected_response): + response = self._service._is_ssh_key_valid(expire_on) + self.assertEqual(response, expected_response) + + @ddt.data((None, None), + ('ssh invalid', None), + ('notadmin:ssh key notadmin', None), + ('Admin:ssh key Admin', 'ssh key Admin'), + ('ssh key google-ssh', None), + ('Admin:s k google-ssh {"userName":"Admin",' + '"expireOn":"1018-12-04T20:12:00+0000"}', + None), + ('Admin:s k google-ssh {"userName":"b",' + '"expireOn":"3018-12-04T20:12:00+0000"}', + None), + ('Admin:s k google-ssh {"userName":"Admin",' + '"expireOn":"3018-12-04T20:12:00+0000"}', + 's k Admin')) + @ddt.unpack + def test__parse_gce_ssh_key(self, raw_ssh_key, expected_response): + response = self._service._parse_gce_ssh_key(raw_ssh_key) + self.assertEqual(response, expected_response) diff --git a/doc/source/services.rst b/doc/source/services.rst index d5812b6e..ef10e9cd 100644 --- a/doc/source/services.rst +++ b/doc/source/services.rst @@ -433,3 +433,37 @@ Capabilities: Config options for `vmwareguestinfo` section: * vmware_rpctool_path (string: "%ProgramFiles%/VMware/VMware Tools/rpctool.exe") + + +Google Compute Engine Service +----------------------------- + +.. class:: cloudbaseinit.metadata.services.gceservice.GCEService + +`GCE `_ metadata service provides +the metadata for instances running on Google Compute Engine. + +GCE metadata is offered via an internal HTTP metadata endpoint, reachable at the magic URL +`http://metadata.google.internal/computeMetadata/v1/`. More information can be found in the GCE +metadata `documents `_. + +To provide userdata to be executed by the instance (in cloud-config format, for example), use the +user-data and user-data-encoding instance metadata keys. + +Capabilities: + + * instance id + * hostname + * public keys + * user data + +Config options for `gce` section: + + * metadata_base_url (string: http://metadata.google.internal/computeMetadata/v1/") + * https_allow_insecure (bool: False) + * https_ca_bundle (string: None) + +Config options for `default` section: + + * retry_count (integer: 5) + * retry_count_interval (integer: 4)