Add Placement helper

This patch added Placement to Watcher
We plan to improve the data model and strategies in
the future specs.

Change-Id: I7141459eef66557cd5d525b5887bd2a381cdac3f
Implements: blueprint support-placement-api
This commit is contained in:
licanwei 2019-05-24 02:18:55 -07:00
parent 15316a57db
commit b57feba5e8
9 changed files with 607 additions and 1 deletions

View File

@ -0,0 +1,8 @@
---
features:
- |
Added Placement API helper to Watcher. Now Watcher can get information
about resource providers, it can be used for the data model and strategies.
Config group placement_client with options 'api_version', 'interface' and
'region_name' is also added. The default values for 'api_version' and
'interface' are 1.29 and 'public', respectively.

View File

@ -16,6 +16,7 @@ from cinderclient import client as ciclient
from glanceclient import client as glclient from glanceclient import client as glclient
from gnocchiclient import client as gnclient from gnocchiclient import client as gnclient
from ironicclient import client as irclient from ironicclient import client as irclient
from keystoneauth1 import adapter as ka_adapter
from keystoneauth1 import loading as ka_loading from keystoneauth1 import loading as ka_loading
from keystoneclient import client as keyclient from keystoneclient import client as keyclient
from monascaclient import client as monclient from monascaclient import client as monclient
@ -73,6 +74,7 @@ class OpenStackClients(object):
self._monasca = None self._monasca = None
self._neutron = None self._neutron = None
self._ironic = None self._ironic = None
self._placement = None
def _get_keystone_session(self): def _get_keystone_session(self):
auth = ka_loading.load_auth_from_conf_options(CONF, auth = ka_loading.load_auth_from_conf_options(CONF,
@ -262,3 +264,27 @@ class OpenStackClients(object):
region_name=ironic_region_name, region_name=ironic_region_name,
session=self.session) session=self.session)
return self._ironic return self._ironic
@exception.wrap_keystone_exception
def placement(self):
if self._placement:
return self._placement
placement_version = self._get_client_option('placement',
'api_version')
placement_interface = self._get_client_option('placement',
'interface')
placement_region_name = self._get_client_option('placement',
'region_name')
# Set accept header on every request to ensure we notify placement
# service of our response body media type preferences.
headers = {'accept': 'application/json'}
self._placement = ka_adapter.Adapter(
session=self.session,
service_type='placement',
default_microversion=placement_version,
interface=placement_interface,
region_name=placement_region_name,
additional_headers=headers)
return self._placement

View File

@ -0,0 +1,179 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
from oslo_log import log as logging
from watcher.common import clients
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
class PlacementHelper(object):
def __init__(self, osc=None):
""":param osc: an OpenStackClients instance"""
self.osc = osc if osc else clients.OpenStackClients()
self._placement = self.osc.placement()
def get(self, url):
return self._placement.get(url, raise_exc=False)
@staticmethod
def get_error_msg(resp):
json_resp = resp.json()
# https://developer.openstack.org/api-ref/placement/#errors
if 'errors' in json_resp:
error_msg = json_resp['errors'][0].get('detail')
else:
error_msg = resp.text
return error_msg
def get_resource_providers(self, rp_name=None):
"""Calls the placement API for a resource provider record.
:param rp_name: Name of the resource provider, if None,
list all resource providers.
:return: A list of resource providers information
or None if the resource provider doesn't exist.
"""
url = '/resource_providers'
if rp_name:
url += '?name=%s' % rp_name
resp = self.get(url)
if resp.status_code == 200:
json_resp = resp.json()
return json_resp['resource_providers']
if rp_name:
msg = "Failed to get resource provider %(name)s. "
else:
msg = "Failed to get all resource providers. "
msg += "Got %(status_code)d: %(err_text)s."
args = {
'name': rp_name,
'status_code': resp.status_code,
'err_text': self.get_error_msg(resp),
}
LOG.error(msg, args)
def get_inventories(self, rp_uuid):
"""Calls the placement API to get resource inventory information.
:param rp_uuid: UUID of the resource provider to get.
:return: A dictionary of inventories keyed by resource classes.
"""
url = '/resource_providers/%s/inventories' % rp_uuid
resp = self.get(url)
if resp.status_code == 200:
json = resp.json()
return json['inventories']
msg = ("Failed to get resource provider %(rp_uuid) inventories. "
"Got %(status_code)d: %(err_text)s.")
args = {
'rp_uuid': rp_uuid,
'status_code': resp.status_code,
'err_text': self.get_error_msg(resp),
}
LOG.error(msg, args)
def get_provider_traits(self, rp_uuid):
"""Queries the placement API for a resource provider's traits.
:param rp_uuid: UUID of the resource provider to grab traits for.
:return: A list of traits.
"""
resp = self.get("/resource_providers/%s/traits" % rp_uuid)
if resp.status_code == 200:
json = resp.json()
return json['traits']
msg = ("Failed to get resource provider %(rp_uuid) traits. "
"Got %(status_code)d: %(err_text)s.")
args = {
'rp_uuid': rp_uuid,
'status_code': resp.status_code,
'err_text': self.get_error_msg(resp),
}
LOG.error(msg, args)
def get_allocations_for_consumer(self, consumer_uuid):
"""Retrieves the allocations for a specific consumer.
:param consumer_uuid: the UUID of the consumer resource.
:return: A dictionary of allocation records keyed by resource
provider uuid.
"""
url = '/allocations/%s' % consumer_uuid
resp = self.get(url)
if resp.status_code == 200:
json = resp.json()
return json['allocations']
msg = ("Failed to get allocations for consumer %(c_uuid). "
"Got %(status_code)d: %(err_text)s.")
args = {
'c_uuid': consumer_uuid,
'status_code': resp.status_code,
'err_text': self.get_error_msg(resp),
}
LOG.error(msg, args)
def get_usages_for_resource_provider(self, rp_uuid):
"""Retrieves the usages for a specific provider.
:param rp_uuid: The UUID of the provider.
:return: A dictionary that describes how much each class of
resource is being consumed on this resource provider.
"""
url = '/resource_providers/%s/usages' % rp_uuid
resp = self.get(url)
if resp.status_code == 200:
json = resp.json()
return json['usages']
msg = ("Failed to get resource provider %(rp_uuid) usages. "
"Got %(status_code)d: %(err_text)s.")
args = {
'rp_uuid': rp_uuid,
'status_code': resp.status_code,
'err_text': self.get_error_msg(resp),
}
LOG.error(msg, args)
def get_candidate_providers(self, resources):
"""Returns a dictionary of resource provider summaries.
:param resources: A comma-separated list of strings indicating
an amount of resource of a specified class that
providers in each allocation request must collectively
have the capacity and availability to serve:
resources=VCPU:4,DISK_GB:64,MEMORY_MB:2048
:returns: A dict, keyed by resource provider UUID, which can
provide the required resources.
"""
url = "/allocation_candidates?%s" % resources
resp = self.get(url)
if resp.status_code == 200:
data = resp.json()
return data['provider_summaries']
args = {
'resource_request': resources,
'status_code': resp.status_code,
'err_text': self.get_error_msg(resp),
}
msg = ("Failed to get allocation candidates from placement "
"API for resources: %(resource_request)s\n"
"Got %(status_code)d: %(err_text)s.")
LOG.error(msg, args)

View File

@ -37,6 +37,7 @@ from watcher.conf import monasca_client
from watcher.conf import neutron_client from watcher.conf import neutron_client
from watcher.conf import nova_client from watcher.conf import nova_client
from watcher.conf import paths from watcher.conf import paths
from watcher.conf import placement_client
from watcher.conf import planner from watcher.conf import planner
from watcher.conf import service from watcher.conf import service
@ -62,3 +63,4 @@ neutron_client.register_opts(CONF)
clients_auth.register_opts(CONF) clients_auth.register_opts(CONF)
ironic_client.register_opts(CONF) ironic_client.register_opts(CONF)
collector.register_opts(CONF) collector.register_opts(CONF)
placement_client.register_opts(CONF)

View File

@ -0,0 +1,41 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
placement_group = cfg.OptGroup(
'placement_client',
title='Placement Service Options',
help="Configuration options for connecting to the placement API service")
placement_opts = [
cfg.StrOpt('api_version',
default='1.29',
help='microversion of placement API when using '
'placement service.'),
cfg.StrOpt('interface',
default='public',
choices=['internal', 'public', 'admin'],
help='Type of endpoint when using placement service.'),
cfg.StrOpt('region_name',
help='Region in Identity service catalog to use for '
'communication with the OpenStack service.')]
def register_opts(conf):
conf.register_group(placement_group)
conf.register_opts(placement_opts, group=placement_group)
def list_opts():
return [(placement_group.name, placement_opts)]

View File

@ -19,6 +19,7 @@ from gnocchiclient import client as gnclient
from gnocchiclient.v1 import client as gnclient_v1 from gnocchiclient.v1 import client as gnclient_v1
from ironicclient import client as irclient from ironicclient import client as irclient
from ironicclient.v1 import client as irclient_v1 from ironicclient.v1 import client as irclient_v1
from keystoneauth1 import adapter as ka_adapter
from keystoneauth1 import loading as ka_loading from keystoneauth1 import loading as ka_loading
import mock import mock
from monascaclient import client as monclient from monascaclient import client as monclient
@ -459,3 +460,17 @@ class TestClients(base.TestCase):
ironic = osc.ironic() ironic = osc.ironic()
ironic_cached = osc.ironic() ironic_cached = osc.ironic()
self.assertEqual(ironic, ironic_cached) self.assertEqual(ironic, ironic_cached)
@mock.patch.object(ka_adapter, 'Adapter')
@mock.patch.object(clients.OpenStackClients, 'session')
def test_clients_placement(self, mock_session, mock_call):
osc = clients.OpenStackClients()
osc.placement()
headers = {'accept': 'application/json'}
mock_call.assert_called_once_with(
session=mock_session,
service_type='placement',
default_microversion=CONF.placement_client.api_version,
interface=CONF.placement_client.interface,
region_name=CONF.placement_client.region_name,
additional_headers=headers)

View File

@ -0,0 +1,312 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from watcher.common import placement_helper
from watcher.tests import base
from watcher.tests import fakes as fake_requests
from keystoneauth1 import loading as ka_loading
from oslo_config import cfg
from oslo_serialization import jsonutils
from oslo_utils import uuidutils
CONF = cfg.CONF
@mock.patch('keystoneauth1.session.Session.request')
class TestPlacementHelper(base.TestCase):
def setUp(self):
super(TestPlacementHelper, self).setUp()
_AUTH_CONF_GROUP = 'watcher_clients_auth'
ka_loading.register_auth_conf_options(CONF, _AUTH_CONF_GROUP)
ka_loading.register_session_conf_options(CONF, _AUTH_CONF_GROUP)
self.client = placement_helper.PlacementHelper()
self.fake_err_msg = {
'errors': [{
'detail': 'The resource could not be found.',
}]
}
def _add_default_kwargs(self, kwargs):
kwargs['endpoint_filter'] = {
'service_type': 'placement',
'interface': CONF.placement_client.interface}
kwargs['headers'] = {'accept': 'application/json'}
kwargs['microversion'] = CONF.placement_client.api_version
kwargs['raise_exc'] = False
def _assert_keystone_called_once(self, kss_req, url, method, **kwargs):
self._add_default_kwargs(kwargs)
# request method has added param rate_semaphore since Stein cycle
if 'rate_semaphore' in kss_req.call_args[1]:
kwargs['rate_semaphore'] = mock.ANY
kss_req.assert_called_once_with(url, method, **kwargs)
def test_get(self, kss_req):
kss_req.return_value = fake_requests.FakeResponse(200)
url = '/resource_providers'
resp = self.client.get(url)
self.assertEqual(200, resp.status_code)
self._assert_keystone_called_once(kss_req, url, 'GET')
def test_get_resource_providers_OK(self, kss_req):
rp_name = 'compute'
rp_uuid = uuidutils.generate_uuid()
parent_uuid = uuidutils.generate_uuid()
fake_rp = [{'uuid': rp_uuid,
'name': rp_name,
'generation': 0,
'parent_provider_uuid': parent_uuid}]
mock_json_data = {
'resource_providers': fake_rp
}
kss_req.return_value = fake_requests.FakeResponse(
200, content=jsonutils.dump_as_bytes(mock_json_data))
result = self.client.get_resource_providers(rp_name)
expected_url = '/resource_providers?name=compute'
self._assert_keystone_called_once(kss_req, expected_url, 'GET')
self.assertEqual(fake_rp, result)
def test_get_resource_providers_no_rp_OK(self, kss_req):
rp_name = None
rp_uuid = uuidutils.generate_uuid()
parent_uuid = uuidutils.generate_uuid()
fake_rp = [{'uuid': rp_uuid,
'name': 'compute',
'generation': 0,
'parent_provider_uuid': parent_uuid}]
mock_json_data = {
'resource_providers': fake_rp
}
kss_req.return_value = fake_requests.FakeResponse(
200, content=jsonutils.dump_as_bytes(mock_json_data))
result = self.client.get_resource_providers(rp_name)
expected_url = '/resource_providers'
self._assert_keystone_called_once(kss_req, expected_url, 'GET')
self.assertEqual(fake_rp, result)
def test_get_resource_providers_fail(self, kss_req):
rp_name = 'compute'
kss_req.return_value = fake_requests.FakeResponse(
400, content=jsonutils.dump_as_bytes(self.fake_err_msg))
result = self.client.get_resource_providers(rp_name)
self.assertIsNone(result)
def test_get_inventories_OK(self, kss_req):
rp_uuid = uuidutils.generate_uuid()
fake_inventories = {
"DISK_GB": {
"allocation_ratio": 1.0,
"max_unit": 35,
"min_unit": 1,
"reserved": 0,
"step_size": 1,
"total": 35
},
"MEMORY_MB": {
"allocation_ratio": 1.5,
"max_unit": 5825,
"min_unit": 1,
"reserved": 512,
"step_size": 1,
"total": 5825
},
"VCPU": {
"allocation_ratio": 16.0,
"max_unit": 4,
"min_unit": 1,
"reserved": 0,
"step_size": 1,
"total": 4
},
}
mock_json_data = {
'inventories': fake_inventories,
"resource_provider_generation": 7
}
kss_req.return_value = fake_requests.FakeResponse(
200, content=jsonutils.dump_as_bytes(mock_json_data))
result = self.client.get_inventories(rp_uuid)
expected_url = '/resource_providers/%s/inventories' % rp_uuid
self._assert_keystone_called_once(kss_req, expected_url, 'GET')
self.assertEqual(fake_inventories, result)
def test_get_inventories_fail(self, kss_req):
rp_uuid = uuidutils.generate_uuid()
kss_req.return_value = fake_requests.FakeResponse(
404, content=jsonutils.dump_as_bytes(self.fake_err_msg))
result = self.client.get_inventories(rp_uuid)
self.assertIsNone(result)
def test_get_provider_traits_OK(self, kss_req):
rp_uuid = uuidutils.generate_uuid()
fake_traits = ["CUSTOM_HW_FPGA_CLASS1",
"CUSTOM_HW_FPGA_CLASS3"]
mock_json_data = {
'traits': fake_traits,
"resource_provider_generation": 7
}
kss_req.return_value = fake_requests.FakeResponse(
200, content=jsonutils.dump_as_bytes(mock_json_data))
result = self.client.get_provider_traits(rp_uuid)
expected_url = '/resource_providers/%s/traits' % rp_uuid
self._assert_keystone_called_once(kss_req, expected_url, 'GET')
self.assertEqual(fake_traits, result)
def test_get_provider_traits_fail(self, kss_req):
rp_uuid = uuidutils.generate_uuid()
kss_req.return_value = fake_requests.FakeResponse(
404, content=jsonutils.dump_as_bytes(self.fake_err_msg))
result = self.client.get_provider_traits(rp_uuid)
self.assertIsNone(result)
def test_get_allocations_for_consumer_OK(self, kss_req):
c_uuid = uuidutils.generate_uuid()
fake_allocations = {
"92637880-2d79-43c6-afab-d860886c6391": {
"generation": 2,
"resources": {
"DISK_GB": 5
}
},
"ba8e1ef8-7fa3-41a4-9bb4-d7cb2019899b": {
"generation": 8,
"resources": {
"MEMORY_MB": 512,
"VCPU": 2
}
}
}
mock_json_data = {
'allocations': fake_allocations,
"consumer_generation": 1,
"project_id": "7e67cbf7-7c38-4a32-b85b-0739c690991a",
"user_id": "067f691e-725a-451a-83e2-5c3d13e1dffc"
}
kss_req.return_value = fake_requests.FakeResponse(
200, content=jsonutils.dump_as_bytes(mock_json_data))
result = self.client.get_allocations_for_consumer(c_uuid)
expected_url = '/allocations/%s' % c_uuid
self._assert_keystone_called_once(kss_req, expected_url, 'GET')
self.assertEqual(fake_allocations, result)
def test_get_allocations_for_consumer_fail(self, kss_req):
c_uuid = uuidutils.generate_uuid()
kss_req.return_value = fake_requests.FakeResponse(
404, content=jsonutils.dump_as_bytes(self.fake_err_msg))
result = self.client.get_allocations_for_consumer(c_uuid)
self.assertIsNone(result)
def test_get_usages_for_resource_provider_OK(self, kss_req):
rp_uuid = uuidutils.generate_uuid()
fake_usages = {
"DISK_GB": 1,
"MEMORY_MB": 512,
"VCPU": 1
}
mock_json_data = {
'usages': fake_usages,
"resource_provider_generation": 7
}
kss_req.return_value = fake_requests.FakeResponse(
200, content=jsonutils.dump_as_bytes(mock_json_data))
result = self.client.get_usages_for_resource_provider(rp_uuid)
expected_url = '/resource_providers/%s/usages' % rp_uuid
self._assert_keystone_called_once(kss_req, expected_url, 'GET')
self.assertEqual(fake_usages, result)
def test_get_usages_for_resource_provider_fail(self, kss_req):
rp_uuid = uuidutils.generate_uuid()
kss_req.return_value = fake_requests.FakeResponse(
404, content=jsonutils.dump_as_bytes(self.fake_err_msg))
result = self.client.get_usages_for_resource_provider(rp_uuid)
self.assertIsNone(result)
def test_get_candidate_providers_OK(self, kss_req):
resources = 'VCPU:4,DISK_GB:64,MEMORY_MB:2048'
fake_provider_summaries = {
"a99bad54-a275-4c4f-a8a3-ac00d57e5c64": {
"resources": {
"DISK_GB": {
"used": 0,
"capacity": 1900
},
},
"traits": ["MISC_SHARES_VIA_AGGREGATE"],
"parent_provider_uuid": None,
"root_provider_uuid": "a99bad54-a275-4c4f-a8a3-ac00d57e5c64"
},
"35791f28-fb45-4717-9ea9-435b3ef7c3b3": {
"resources": {
"VCPU": {
"used": 0,
"capacity": 384
},
"MEMORY_MB": {
"used": 0,
"capacity": 196608
},
},
"traits": ["HW_CPU_X86_SSE2", "HW_CPU_X86_AVX2"],
"parent_provider_uuid": None,
"root_provider_uuid": "35791f28-fb45-4717-9ea9-435b3ef7c3b3"
},
}
mock_json_data = {
'provider_summaries': fake_provider_summaries,
}
kss_req.return_value = fake_requests.FakeResponse(
200, content=jsonutils.dump_as_bytes(mock_json_data))
result = self.client.get_candidate_providers(resources)
expected_url = "/allocation_candidates?%s" % resources
self._assert_keystone_called_once(kss_req, expected_url, 'GET')
self.assertEqual(fake_provider_summaries, result)
def test_get_candidate_providers_fail(self, kss_req):
rp_uuid = uuidutils.generate_uuid()
kss_req.return_value = fake_requests.FakeResponse(
404, content=jsonutils.dump_as_bytes(self.fake_err_msg))
result = self.client.get_candidate_providers(rp_uuid)
self.assertIsNone(result)

View File

@ -33,7 +33,7 @@ class TestListOpts(base.TestCase):
'nova_client', 'glance_client', 'gnocchi_client', 'cinder_client', 'nova_client', 'glance_client', 'gnocchi_client', 'cinder_client',
'ceilometer_client', 'monasca_client', 'ironic_client', 'ceilometer_client', 'monasca_client', 'ironic_client',
'keystone_client', 'neutron_client', 'watcher_clients_auth', 'keystone_client', 'neutron_client', 'watcher_clients_auth',
'collector'] 'collector', 'placement_client']
self.opt_sections = list(dict(opts.list_opts()).keys()) self.opt_sections = list(dict(opts.list_opts()).keys())
def test_run_list_opts(self): def test_run_list_opts(self):

View File

@ -11,6 +11,7 @@
# under the License. # under the License.
import mock import mock
import requests
fakeAuthTokenHeaders = {'X-User-Id': u'773a902f022949619b5c2f32cd89d419', fakeAuthTokenHeaders = {'X-User-Id': u'773a902f022949619b5c2f32cd89d419',
'X-Roles': u'admin, ResellerAdmin, _member_', 'X-Roles': u'admin, ResellerAdmin, _member_',
@ -88,3 +89,25 @@ class FakeAuthProtocol(mock.Mock):
super(FakeAuthProtocol, self).__init__(**kwargs) super(FakeAuthProtocol, self).__init__(**kwargs)
self.app = FakeApp() self.app = FakeApp()
self.config = '' self.config = ''
class FakeResponse(requests.Response):
def __init__(self, status_code, content=None, headers=None):
"""A requests.Response that can be used as a mock return_value.
A key feature is that the instance will evaluate to True or False like
a real Response, based on the status_code.
Properties like ok, status_code, text, and content, and methods like
json(), work as expected based on the inputs.
:param status_code: Integer HTTP response code (200, 404, etc.)
:param content: String supplying the payload content of the response.
Using a json-encoded string will make the json() method
behave as expected.
:param headers: Dict of HTTP header values to set.
"""
super(FakeResponse, self).__init__()
self.status_code = status_code
if content:
self._content = content
if headers:
self.headers = headers