openstack/watcher
Code Issues Proposed changes

Add Placement helper

Browse Source 
This patch added Placement to Watcher
We plan to improve the data model and strategies in
the future specs.

Change-Id: I7141459eef66557cd5d525b5887bd2a381cdac3f
Implements: blueprint support-placement-api
This commit is contained in:
licanwei 2019-05-24 02:18:55 -07:00
parent 15316a57db
commit b57feba5e8
9 changed files with 607 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
releasenotes/notes/support-placement-api-58ce6bef1bbbe98a.yaml Normal file
View File

@ -0,0 +1,8 @@
---
features:
- |
Added Placement API helper to Watcher. Now Watcher can get information
about resource providers, it can be used for the data model and strategies.
Config group placement_client with options 'api_version', 'interface' and
'region_name' is also added. The default values for 'api_version' and
'interface' are 1.29 and 'public', respectively.

26
watcher/common/clients.py
View File

@ -16,6 +16,7 @@ from cinderclient import client as ciclient
from glanceclient import client as glclient
from gnocchiclient import client as gnclient
from ironicclient import client as irclient
from keystoneauth1 import adapter as ka_adapter
from keystoneauth1 import loading as ka_loading
from keystoneclient import client as keyclient
from monascaclient import client as monclient
@ -73,6 +74,7 @@ class OpenStackClients(object):
self._monasca = None
self._neutron = None
self._ironic = None
self._placement = None
def _get_keystone_session(self):
auth = ka_loading.load_auth_from_conf_options(CONF,
@ -262,3 +264,27 @@ class OpenStackClients(object):
region_name=ironic_region_name,
session=self.session)
return self._ironic
@exception.wrap_keystone_exception
def placement(self):
if self._placement:
return self._placement
placement_version = self._get_client_option('placement',
'api_version')
placement_interface = self._get_client_option('placement',
'interface')
placement_region_name = self._get_client_option('placement',
'region_name')
# Set accept header on every request to ensure we notify placement
# service of our response body media type preferences.
headers = {'accept': 'application/json'}
self._placement = ka_adapter.Adapter(
session=self.session,
service_type='placement',
default_microversion=placement_version,
interface=placement_interface,
region_name=placement_region_name,
additional_headers=headers)
return self._placement

179
watcher/common/placement_helper.py Normal file
View File

@ -0,0 +1,179 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
from oslo_log import log as logging
from watcher.common import clients
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
class PlacementHelper(object):
def __init__(self, osc=None):
""":param osc: an OpenStackClients instance"""
self.osc = osc if osc else clients.OpenStackClients()
self._placement = self.osc.placement()
def get(self, url):
return self._placement.get(url, raise_exc=False)
@staticmethod
def get_error_msg(resp):
json_resp = resp.json()
# https://developer.openstack.org/api-ref/placement/#errors
if 'errors' in json_resp:
error_msg = json_resp['errors'][0].get('detail')
else:
error_msg = resp.text
return error_msg
def get_resource_providers(self, rp_name=None):
"""Calls the placement API for a resource provider record.
:param rp_name: Name of the resource provider, if None,
list all resource providers.
:return: A list of resource providers information
or None if the resource provider doesn't exist.
"""
url = '/resource_providers'
if rp_name:
url += '?name=%s' % rp_name
resp = self.get(url)
if resp.status_code == 200:
json_resp = resp.json()
return json_resp['resource_providers']
if rp_name:
msg = "Failed to get resource provider %(name)s. "
else:
msg = "Failed to get all resource providers. "
msg += "Got %(status_code)d: %(err_text)s."
args = {
'name': rp_name,
'status_code': resp.status_code,
'err_text': self.get_error_msg(resp),
}
LOG.error(msg, args)
def get_inventories(self, rp_uuid):
"""Calls the placement API to get resource inventory information.
:param rp_uuid: UUID of the resource provider to get.
:return: A dictionary of inventories keyed by resource classes.
"""
url = '/resource_providers/%s/inventories' % rp_uuid
resp = self.get(url)
if resp.status_code == 200:
json = resp.json()
return json['inventories']
msg = ("Failed to get resource provider %(rp_uuid) inventories. "
"Got %(status_code)d: %(err_text)s.")
args = {
'rp_uuid': rp_uuid,
'status_code': resp.status_code,
'err_text': self.get_error_msg(resp),
}
LOG.error(msg, args)
def get_provider_traits(self, rp_uuid):
"""Queries the placement API for a resource provider's traits.
:param rp_uuid: UUID of the resource provider to grab traits for.
:return: A list of traits.
"""
resp = self.get("/resource_providers/%s/traits" % rp_uuid)
if resp.status_code == 200:
json = resp.json()
return json['traits']
msg = ("Failed to get resource provider %(rp_uuid) traits. "
"Got %(status_code)d: %(err_text)s.")
args = {
'rp_uuid': rp_uuid,
'status_code': resp.status_code,
'err_text': self.get_error_msg(resp),
}
LOG.error(msg, args)
def get_allocations_for_consumer(self, consumer_uuid):
"""Retrieves the allocations for a specific consumer.
:param consumer_uuid: the UUID of the consumer resource.
:return: A dictionary of allocation records keyed by resource
provider uuid.
"""
url = '/allocations/%s' % consumer_uuid
resp = self.get(url)
if resp.status_code == 200:
json = resp.json()
return json['allocations']
msg = ("Failed to get allocations for consumer %(c_uuid). "
"Got %(status_code)d: %(err_text)s.")
args = {
'c_uuid': consumer_uuid,
'status_code': resp.status_code,
'err_text': self.get_error_msg(resp),
}
LOG.error(msg, args)
def get_usages_for_resource_provider(self, rp_uuid):
"""Retrieves the usages for a specific provider.
:param rp_uuid: The UUID of the provider.
:return: A dictionary that describes how much each class of
resource is being consumed on this resource provider.
"""
url = '/resource_providers/%s/usages' % rp_uuid
resp = self.get(url)
if resp.status_code == 200:
json = resp.json()
return json['usages']
msg = ("Failed to get resource provider %(rp_uuid) usages. "
"Got %(status_code)d: %(err_text)s.")
args = {
'rp_uuid': rp_uuid,
'status_code': resp.status_code,
'err_text': self.get_error_msg(resp),
}
LOG.error(msg, args)
def get_candidate_providers(self, resources):
"""Returns a dictionary of resource provider summaries.
:param resources: A comma-separated list of strings indicating
an amount of resource of a specified class that
providers in each allocation request must collectively
have the capacity and availability to serve:
resources=VCPU:4,DISK_GB:64,MEMORY_MB:2048
:returns: A dict, keyed by resource provider UUID, which can
provide the required resources.
"""
url = "/allocation_candidates?%s" % resources
resp = self.get(url)
if resp.status_code == 200:
data = resp.json()
return data['provider_summaries']
args = {
'resource_request': resources,
'status_code': resp.status_code,
'err_text': self.get_error_msg(resp),
}
msg = ("Failed to get allocation candidates from placement "
"API for resources: %(resource_request)s\n"
"Got %(status_code)d: %(err_text)s.")
LOG.error(msg, args)

2
watcher/conf/__init__.py
View File

@ -37,6 +37,7 @@ from watcher.conf import monasca_client
from watcher.conf import neutron_client
from watcher.conf import nova_client
from watcher.conf import paths
from watcher.conf import placement_client
from watcher.conf import planner
from watcher.conf import service
@ -62,3 +63,4 @@ neutron_client.register_opts(CONF)
clients_auth.register_opts(CONF)
ironic_client.register_opts(CONF)
collector.register_opts(CONF)
placement_client.register_opts(CONF)

41
watcher/conf/placement_client.py Normal file
View File

@ -0,0 +1,41 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
placement_group = cfg.OptGroup(
'placement_client',
title='Placement Service Options',
help="Configuration options for connecting to the placement API service")
placement_opts = [
cfg.StrOpt('api_version',
default='1.29',
help='microversion of placement API when using '
'placement service.'),
cfg.StrOpt('interface',
default='public',
choices=['internal', 'public', 'admin'],
help='Type of endpoint when using placement service.'),
cfg.StrOpt('region_name',
help='Region in Identity service catalog to use for '
'communication with the OpenStack service.')]
def register_opts(conf):
conf.register_group(placement_group)
conf.register_opts(placement_opts, group=placement_group)
def list_opts():
return [(placement_group.name, placement_opts)]

15
watcher/tests/common/test_clients.py
View File

@ -19,6 +19,7 @@ from gnocchiclient import client as gnclient
from gnocchiclient.v1 import client as gnclient_v1
from ironicclient import client as irclient
from ironicclient.v1 import client as irclient_v1
from keystoneauth1 import adapter as ka_adapter
from keystoneauth1 import loading as ka_loading
import mock
from monascaclient import client as monclient
@ -459,3 +460,17 @@ class TestClients(base.TestCase):
ironic = osc.ironic()
ironic_cached = osc.ironic()
self.assertEqual(ironic, ironic_cached)
@mock.patch.object(ka_adapter, 'Adapter')
@mock.patch.object(clients.OpenStackClients, 'session')
def test_clients_placement(self, mock_session, mock_call):
osc = clients.OpenStackClients()
osc.placement()
headers = {'accept': 'application/json'}
mock_call.assert_called_once_with(
session=mock_session,
service_type='placement',
default_microversion=CONF.placement_client.api_version,
interface=CONF.placement_client.interface,
region_name=CONF.placement_client.region_name,
additional_headers=headers)

312
watcher/tests/common/test_placement_helper.py Normal file
View File

@ -0,0 +1,312 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from watcher.common import placement_helper
from watcher.tests import base
from watcher.tests import fakes as fake_requests
from keystoneauth1 import loading as ka_loading
from oslo_config import cfg
from oslo_serialization import jsonutils
from oslo_utils import uuidutils
CONF = cfg.CONF
@mock.patch('keystoneauth1.session.Session.request')
class TestPlacementHelper(base.TestCase):
def setUp(self):
super(TestPlacementHelper, self).setUp()
_AUTH_CONF_GROUP = 'watcher_clients_auth'
ka_loading.register_auth_conf_options(CONF, _AUTH_CONF_GROUP)
ka_loading.register_session_conf_options(CONF, _AUTH_CONF_GROUP)
self.client = placement_helper.PlacementHelper()
self.fake_err_msg = {
'errors': [{
'detail': 'The resource could not be found.',
}]
}
def _add_default_kwargs(self, kwargs):
kwargs['endpoint_filter'] = {
'service_type': 'placement',
'interface': CONF.placement_client.interface}
kwargs['headers'] = {'accept': 'application/json'}
kwargs['microversion'] = CONF.placement_client.api_version
kwargs['raise_exc'] = False
def _assert_keystone_called_once(self, kss_req, url, method, **kwargs):
self._add_default_kwargs(kwargs)
# request method has added param rate_semaphore since Stein cycle
if 'rate_semaphore' in kss_req.call_args[1]:
kwargs['rate_semaphore'] = mock.ANY
kss_req.assert_called_once_with(url, method, **kwargs)
def test_get(self, kss_req):
kss_req.return_value = fake_requests.FakeResponse(200)
url = '/resource_providers'
resp = self.client.get(url)
self.assertEqual(200, resp.status_code)
self._assert_keystone_called_once(kss_req, url, 'GET')
def test_get_resource_providers_OK(self, kss_req):
rp_name = 'compute'
rp_uuid = uuidutils.generate_uuid()
parent_uuid = uuidutils.generate_uuid()
fake_rp = [{'uuid': rp_uuid,
'name': rp_name,
'generation': 0,
'parent_provider_uuid': parent_uuid}]
mock_json_data = {
'resource_providers': fake_rp
}
kss_req.return_value = fake_requests.FakeResponse(
200, content=jsonutils.dump_as_bytes(mock_json_data))
result = self.client.get_resource_providers(rp_name)
expected_url = '/resource_providers?name=compute'
self._assert_keystone_called_once(kss_req, expected_url, 'GET')
self.assertEqual(fake_rp, result)
def test_get_resource_providers_no_rp_OK(self, kss_req):
rp_name = None
rp_uuid = uuidutils.generate_uuid()
parent_uuid = uuidutils.generate_uuid()
fake_rp = [{'uuid': rp_uuid,
'name': 'compute',
'generation': 0,
'parent_provider_uuid': parent_uuid}]
mock_json_data = {
'resource_providers': fake_rp
}
kss_req.return_value = fake_requests.FakeResponse(
200, content=jsonutils.dump_as_bytes(mock_json_data))
result = self.client.get_resource_providers(rp_name)
expected_url = '/resource_providers'
self._assert_keystone_called_once(kss_req, expected_url, 'GET')
self.assertEqual(fake_rp, result)
def test_get_resource_providers_fail(self, kss_req):
rp_name = 'compute'
kss_req.return_value = fake_requests.FakeResponse(
400, content=jsonutils.dump_as_bytes(self.fake_err_msg))
result = self.client.get_resource_providers(rp_name)
self.assertIsNone(result)
def test_get_inventories_OK(self, kss_req):
rp_uuid = uuidutils.generate_uuid()
fake_inventories = {
"DISK_GB": {
"allocation_ratio": 1.0,
"max_unit": 35,
"min_unit": 1,
"reserved": 0,
"step_size": 1,
"total": 35
},
"MEMORY_MB": {
"allocation_ratio": 1.5,
"max_unit": 5825,
"min_unit": 1,
"reserved": 512,
"step_size": 1,
"total": 5825
},
"VCPU": {
"allocation_ratio": 16.0,
"max_unit": 4,
"min_unit": 1,
"reserved": 0,
"step_size": 1,
"total": 4
},
}
mock_json_data = {
'inventories': fake_inventories,
"resource_provider_generation": 7
}
kss_req.return_value = fake_requests.FakeResponse(
200, content=jsonutils.dump_as_bytes(mock_json_data))
result = self.client.get_inventories(rp_uuid)
expected_url = '/resource_providers/%s/inventories' % rp_uuid
self._assert_keystone_called_once(kss_req, expected_url, 'GET')
self.assertEqual(fake_inventories, result)
def test_get_inventories_fail(self, kss_req):
rp_uuid = uuidutils.generate_uuid()
kss_req.return_value = fake_requests.FakeResponse(
404, content=jsonutils.dump_as_bytes(self.fake_err_msg))
result = self.client.get_inventories(rp_uuid)
self.assertIsNone(result)
def test_get_provider_traits_OK(self, kss_req):
rp_uuid = uuidutils.generate_uuid()
fake_traits = ["CUSTOM_HW_FPGA_CLASS1",
"CUSTOM_HW_FPGA_CLASS3"]
mock_json_data = {
'traits': fake_traits,
"resource_provider_generation": 7
}
kss_req.return_value = fake_requests.FakeResponse(
200, content=jsonutils.dump_as_bytes(mock_json_data))
result = self.client.get_provider_traits(rp_uuid)
expected_url = '/resource_providers/%s/traits' % rp_uuid
self._assert_keystone_called_once(kss_req, expected_url, 'GET')
self.assertEqual(fake_traits, result)
def test_get_provider_traits_fail(self, kss_req):
rp_uuid = uuidutils.generate_uuid()
kss_req.return_value = fake_requests.FakeResponse(
404, content=jsonutils.dump_as_bytes(self.fake_err_msg))
result = self.client.get_provider_traits(rp_uuid)
self.assertIsNone(result)
def test_get_allocations_for_consumer_OK(self, kss_req):
c_uuid = uuidutils.generate_uuid()
fake_allocations = {
"92637880-2d79-43c6-afab-d860886c6391": {
"generation": 2,
"resources": {
"DISK_GB": 5
}
},
"ba8e1ef8-7fa3-41a4-9bb4-d7cb2019899b": {
"generation": 8,
"resources": {
"MEMORY_MB": 512,
"VCPU": 2
}
}
}
mock_json_data = {
'allocations': fake_allocations,
"consumer_generation": 1,
"project_id": "7e67cbf7-7c38-4a32-b85b-0739c690991a",
"user_id": "067f691e-725a-451a-83e2-5c3d13e1dffc"
}
kss_req.return_value = fake_requests.FakeResponse(
200, content=jsonutils.dump_as_bytes(mock_json_data))
result = self.client.get_allocations_for_consumer(c_uuid)
expected_url = '/allocations/%s' % c_uuid
self._assert_keystone_called_once(kss_req, expected_url, 'GET')
self.assertEqual(fake_allocations, result)
def test_get_allocations_for_consumer_fail(self, kss_req):
c_uuid = uuidutils.generate_uuid()
kss_req.return_value = fake_requests.FakeResponse(
404, content=jsonutils.dump_as_bytes(self.fake_err_msg))
result = self.client.get_allocations_for_consumer(c_uuid)
self.assertIsNone(result)
def test_get_usages_for_resource_provider_OK(self, kss_req):
rp_uuid = uuidutils.generate_uuid()
fake_usages = {
"DISK_GB": 1,
"MEMORY_MB": 512,
"VCPU": 1
}
mock_json_data = {
'usages': fake_usages,
"resource_provider_generation": 7
}
kss_req.return_value = fake_requests.FakeResponse(
200, content=jsonutils.dump_as_bytes(mock_json_data))
result = self.client.get_usages_for_resource_provider(rp_uuid)
expected_url = '/resource_providers/%s/usages' % rp_uuid
self._assert_keystone_called_once(kss_req, expected_url, 'GET')
self.assertEqual(fake_usages, result)
def test_get_usages_for_resource_provider_fail(self, kss_req):
rp_uuid = uuidutils.generate_uuid()
kss_req.return_value = fake_requests.FakeResponse(
404, content=jsonutils.dump_as_bytes(self.fake_err_msg))
result = self.client.get_usages_for_resource_provider(rp_uuid)
self.assertIsNone(result)
def test_get_candidate_providers_OK(self, kss_req):
resources = 'VCPU:4,DISK_GB:64,MEMORY_MB:2048'
fake_provider_summaries = {
"a99bad54-a275-4c4f-a8a3-ac00d57e5c64": {
"resources": {
"DISK_GB": {
"used": 0,
"capacity": 1900
},
},
"traits": ["MISC_SHARES_VIA_AGGREGATE"],
"parent_provider_uuid": None,
"root_provider_uuid": "a99bad54-a275-4c4f-a8a3-ac00d57e5c64"
},
"35791f28-fb45-4717-9ea9-435b3ef7c3b3": {
"resources": {
"VCPU": {
"used": 0,
"capacity": 384
},
"MEMORY_MB": {
"used": 0,
"capacity": 196608
},
},
"traits": ["HW_CPU_X86_SSE2", "HW_CPU_X86_AVX2"],
"parent_provider_uuid": None,
"root_provider_uuid": "35791f28-fb45-4717-9ea9-435b3ef7c3b3"
},
}
mock_json_data = {
'provider_summaries': fake_provider_summaries,
}
kss_req.return_value = fake_requests.FakeResponse(
200, content=jsonutils.dump_as_bytes(mock_json_data))
result = self.client.get_candidate_providers(resources)
expected_url = "/allocation_candidates?%s" % resources
self._assert_keystone_called_once(kss_req, expected_url, 'GET')
self.assertEqual(fake_provider_summaries, result)
def test_get_candidate_providers_fail(self, kss_req):
rp_uuid = uuidutils.generate_uuid()
kss_req.return_value = fake_requests.FakeResponse(
404, content=jsonutils.dump_as_bytes(self.fake_err_msg))
result = self.client.get_candidate_providers(rp_uuid)
self.assertIsNone(result)

2
watcher/tests/conf/test_list_opts.py
View File

@ -33,7 +33,7 @@ class TestListOpts(base.TestCase):
'nova_client', 'glance_client', 'gnocchi_client', 'cinder_client',
'ceilometer_client', 'monasca_client', 'ironic_client',
'keystone_client', 'neutron_client', 'watcher_clients_auth',
'collector']
'collector', 'placement_client']
self.opt_sections = list(dict(opts.list_opts()).keys())
def test_run_list_opts(self):

23
watcher/tests/fakes.py
View File

@ -11,6 +11,7 @@
# under the License.
import mock
import requests
fakeAuthTokenHeaders = {'X-User-Id': u'773a902f022949619b5c2f32cd89d419',
'X-Roles': u'admin, ResellerAdmin, _member_',
@ -88,3 +89,25 @@ class FakeAuthProtocol(mock.Mock):
super(FakeAuthProtocol, self).__init__(**kwargs)
self.app = FakeApp()
self.config = ''
class FakeResponse(requests.Response):
def __init__(self, status_code, content=None, headers=None):
"""A requests.Response that can be used as a mock return_value.
A key feature is that the instance will evaluate to True or False like
a real Response, based on the status_code.
Properties like ok, status_code, text, and content, and methods like
json(), work as expected based on the inputs.
:param status_code: Integer HTTP response code (200, 404, etc.)
:param content: String supplying the payload content of the response.
Using a json-encoded string will make the json() method
behave as expected.
:param headers: Dict of HTTP header values to set.
"""
super(FakeResponse, self).__init__()
self.status_code = status_code
if content:
self._content = content
if headers:
self.headers = headers