airship/drydock
Code Issues Proposed changes

Add locking around DesignState change methods to prepare

Browse Source 
for multithreaded access

Support design and build change merging instead of just
replacing. Includes basic unit test.

Add readme for model design
This commit is contained in:
Scott Hussey 2017-03-21 16:30:35 -05:00
parent f9cfc23997
commit 37daf1f95f
9 changed files with 294 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
helm_drydock/error.py
View File

@ -14,3 +14,6 @@
class DesignError(Exception):
pass
class StateError(Exception):
pass

10
helm_drydock/model/hostprofile.py
View File

@ -146,13 +146,9 @@ class HostProfile(object):
self.applied['interfaces'] = HostInterface.merge_lists(
self.design['interfaces'], parent.applied['interfaces'])
for i in self.applied.get('interfaces', []):
i.ensure_applied_data()
self.applied['partitions'] = HostPartition.merge_lists(
self.design['partitions'], parent.applied['partitions'])
for p in self.applied.get('partitions', []):
p. ensure_applied_data()
return
@ -293,11 +289,11 @@ class HostInterface(object):
elif len(parent_list) > 0 and len(child_list) > 0:
parent_interfaces = []
for i in parent_list:
parent_name = i.device_name
parent_name = i.get_name()
parent_interfaces.append(parent_name)
add = True
for j in child_list:
if j.device_name == ("!" + parent_name):
if j.get_name() == ("!" + parent_name):
add = False
break
elif j.device_name == parent_name:
@ -312,7 +308,6 @@ class HostInterface(object):
in i.applied.get('hardware_slaves', [])
if ("!" + x) not in j.design.get(
'hardware_slaves', [])]
s = list(s)
s.extend(
[x for x
@ -325,7 +320,6 @@ class HostInterface(object):
in i.applied.get('networks',[])
if ("!" + x) not in j.design.get(
'networks', [])]
n = list(n)
n.extend(
[x for x

3
helm_drydock/model/hwprofile.py
View File

@ -67,6 +67,9 @@ class HardwareProfile(object):
return
def get_name(self):
return self.name
def resolve_alias(self, alias_type, alias):
selector = {}
for d in self.devices:

5
helm_drydock/model/network.py
View File

@ -58,6 +58,8 @@ class NetworkLink(object):
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')
def get_name(self):
return self.name
class Network(object):
@ -98,6 +100,9 @@ class Network(object):
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')
def get_name(self):
return self.name
class NetworkAddressRange(object):

37
helm_drydock/model/readme.md Normal file
View File

@ -0,0 +1,37 @@
# Drydock Model #
Models for the drydock design parts and subparts
## Features ##
### Inheritance ###
Drydock supports inheritance in the design data model.
Currently this only supports BaremetalNode inheriting from HostProfile and
HostProfile inheriting from HostProfile.
Inheritance rules:
1. A child overrides a parent for part and subpart attributes
2. For attributes that are lists, the parent list and child list
are merged.
3. A child can remove a list member by prefixing the value with '!'
4. For lists of subparts (i.e. HostInterface and HostPartition) if
there is a member in the parent list and child list with the same name
(as defined by the get_name() method), the child member inherits from
the parent member. The '!' prefix applies here for deleting a member
based on the name.
### Phased Data ###
In other words, as a modeled object goes from design to apply
to build the model keeps the data separated to retain reference
values and provide context around particular attribute values.
* Design - The data ingested from sources such as Formation
* Apply - Computing inheritance of design data to render an effective site design
* Build - Maintaining actions taken to implement the design and the results
Currently only applies to BaremetalNodes as no other design parts
flow through the build process.

3
helm_drydock/model/site.py
View File

@ -60,6 +60,9 @@ class Site(object):
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')
def get_name(self):
return self.name
def start_build(self):
if self.build.get('status', '') == '':
self.build['status'] = SiteStatus.Unknown

4
helm_drydock/orchestrator/designdata.py
View File

@ -91,9 +91,7 @@ class DesignStateClient(object):
site_copy = deepcopy(site_root)
for n in site_copy.baremetal_nodes:
n.apply_host_profile(site_copy)
n.apply_hardware_profile(site_copy)
n.apply_network_connections(site_copy)
n.compile_applied_model(site_copy)
return site_copy
"""

172
helm_drydock/statemgmt/__init__.py
View File

@ -14,6 +14,7 @@
from copy import deepcopy
from datetime import datetime
from datetime import timezone
from threading import Lock
import uuid
@ -23,16 +24,20 @@ import helm_drydock.model.network as network
import helm_drydock.model.site as site
import helm_drydock.model.hwprofile as hwprofile
from helm_drydock.error import DesignError
from helm_drydock.error import DesignError, StateError
class DesignState(object):
def __init__(self):
self.design_base = None
self.design_base_lock = Lock()
self.design_changes = []
self.design_changes_lock = Lock()
self.builds = []
self.builds_lock = Lock()
return
# TODO Need to lock a design base or change once implementation
@ -45,14 +50,27 @@ class DesignState(object):
def post_design_base(self, site_design):
if site_design is not None and isinstance(site_design, SiteDesign):
my_lock = self.design_base_lock.acquire(blocking=True,
timeout=10)
if my_lock:
self.design_base = deepcopy(site_design)
self.design_base_lock.release()
return True
raise StateError("Could not acquire lock")
else:
raise DesignError("Design change must be a SiteDesign instance")
def put_design_base(self, site_design):
# TODO Support merging
if site_design is not None and isinstance(site_design, SiteDesign):
self.design_base = deepcopy(site_design)
my_lock = self.design_base_lock.acquire(blocking=True,
timeout=10)
if my_lock:
self.design_base.merge_updates(site_design)
self.design_base_lock.release()
return True
raise StateError("Could not acquire lock")
else:
raise DesignError("Design base must be a SiteDesign instance")
def get_design_change(self, changeid):
match = [x for x in self.design_changes if x.changeid == changeid]
@ -64,27 +82,35 @@ class DesignState(object):
def post_design_change(self, site_design):
if site_design is not None and isinstance(site_design, SiteDesign):
my_lock = self.design_changes_lock.acquire(block=True,
timeout=10)
if my_lock:
exists = [(x) for x
in self.design_changes
if x.changeid == site_design.changeid]
if len(exists) > 0:
self.design_changs_lock.release()
raise DesignError("Existing change %s found" %
(site_design.changeid))
self.design_changes.append(deepcopy(site_design))
self.design_changes_lock.release()
return True
raise StateError("Could not acquire lock")
else:
raise DesignError("Design change must be a SiteDesign instance")
def put_design_change(self, site_design):
# TODO Support merging
if site_design is not None and isinstance(site_design, SiteDesign):
design_copy = deepcopy(site_design)
self.design_changes = [design_copy
if x.changeid == design_copy.changeid
else x
for x
in self.design_changes]
my_lock = self.design_changes_lock.acquire(block=True,
timeout=10)
if my_lock:
changeid = site_design.changeid
for c in self.design_changes:
if c.changeid == changeid:
c.merge_updates(site_design)
return True
raise StateError("Could not acquire lock")
else:
raise DesignError("Design change must be a SiteDesign instance")
@ -108,23 +134,48 @@ class DesignState(object):
def post_build(self, site_build):
if site_build is not None and isinstance(site_build, SiteBuild):
my_lock = self.builds_lock.acquire(block=True, timeout=10)
if my_lock:
exists = [b for b in self.builds
if b.build_id == site_build.build_id]
if len(exists) > 0:
self.builds_lock.release()
raise DesignError("Already a site build with ID %s" %
(str(site_build.build_id)))
else:
self.builds.append(deepcopy(site_build))
self.builds_lock.release()
return True
raise StateError("Could not acquire lock")
else:
raise DesignError("Design change must be a SiteDesign instance")
def put_build(self, site_build):
if site_build is not None and isinstance(site_build, SiteBuild):
my_lock = self.builds_lock.acquire(block=True, timeout=10)
if my_lock:
buildid = site_build.buildid
for b in self.builds:
if b.buildid == buildid:
b.merge_updates(site_build)
self.builds_lock.release()
return True
self.builds_lock.release()
return False
raise StateError("Could not acquire lock")
else:
raise DesignError("Design change must be a SiteDesign instance")
class SiteDesign(object):
def __init__(self, ischange=False):
def __init__(self, ischange=False, changeid=None):
if ischange:
if changeid is not None:
self.changeid = changeid
else:
self.changeid = uuid.uuid4()
else:
# Base design
self.changeid = 0
self.sites = []
@ -140,6 +191,17 @@ class SiteDesign(object):
self.sites.append(new_site)
def update_site(self, update):
if update is None or not isinstance(update, site.Site):
raise DesignError("Invalid Site model")
for i, s in enumerate(self.sites):
if s.get_name() == update.get_name():
self.sites[i] = deepcopy(update)
return True
return False
def get_sites(self):
return self.sites
@ -156,6 +218,17 @@ class SiteDesign(object):
self.networks.append(new_network)
def update_network(self, update):
if update is None or not isinstance(update, network.Network):
raise DesignError("Invalid Network model")
for i, n in enumerate(self.networks):
if n.get_name() == update.get_name():
self.networks[i] = deepcopy(update)
return True
return False
def get_networks(self):
return self.networks
@ -174,6 +247,17 @@ class SiteDesign(object):
self.network_links.append(new_network_link)
def update_network_link(self, update):
if update is None or not isinstance(update, network.NetworkLink):
raise DesignError("Invalid NetworkLink model")
for i, n in enumerate(self.network_links):
if n.get_name() == update.get_name():
self.network_links[i] = deepcopy(update)
return True
return False
def get_network_links(self):
return self.network_links
@ -192,6 +276,17 @@ class SiteDesign(object):
self.host_profiles.append(new_host_profile)
def update_host_profile(self, update):
if update is None or not isinstance(update, hostprofile.HostProfile):
raise DesignError("Invalid HostProfile model")
for i, h in enumerate(self.host_profiles):
if h.get_name() == update.get_name():
self.host_profiles[i] = deepcopy(h)
return True
return False
def get_host_profiles(self):
return self.host_profiles
@ -210,6 +305,17 @@ class SiteDesign(object):
self.hardware_profiles.append(new_hardware_profile)
def update_hardware_profile(self, update):
if update is None or not isinstance(update, hwprofile.HardwareProfile):
raise DesignError("Invalid HardwareProfile model")
for i, h in enumerate(self.hardware_profiles):
if h.get_name() == update.get_name():
self.hardware_profiles[i] = deepcopy(h)
return True
return False
def get_hardware_profiles(self):
return self.hardware_profiles
@ -228,6 +334,17 @@ class SiteDesign(object):
self.baremetal_nodes.append(new_baremetal_node)
def update_baremetal_node(self, update):
if (update is None or not isinstance(update, node.BaremetalNode)):
raise DesignError("Invalid BaremetalNode model")
for i, b in enumerate(self.baremetal_nodes):
if b.get_name() == update.get_name():
self.baremetal_nodes[i] = deepcopy(b)
return True
return False
def get_baremetal_nodes(self):
return self.baremetal_nodes
@ -239,6 +356,33 @@ class SiteDesign(object):
raise DesignError("BaremetalNode %s not found in design state"
% node_name)
# Only merge the design parts included in the updated site
# design. Changes are merged at the part level, not for fields
# within a design part
#
# TODO convert update_* methods to use exceptions and convert to try block
def merge_updates(self, updates):
if updates is not None and isinstance(updates, SiteDesign):
if updates.changeid == self.changeid:
for u in updates.sites:
if not self.update_site(u):
self.add_site(u)
for u in updates.networks:
if not self.update_network(u):
self.add_network(u)
for u in updates.network_links:
if not self.update_network_link(u):
self.add_network_link(u)
for u in updates.host_profiles:
if not self.update_host_profile(u):
self.add_host_profile(u)
for u in updates.hardware_profiles:
if not self.update_hardware_profile(u):
self.add_hardware_profile(u)
for u in updates.baremetal_nodes:
if not self.update_baremetal_node(u):
self.add_baremetal_node(u)
class SiteBuild(SiteDesign):
@ -246,9 +390,9 @@ class SiteBuild(SiteDesign):
super(SiteBuild, self).__init__()
if build_id is None:
self.build_id = datetime.datetime.now(timezone.utc).timestamp()
self.buildid = datetime.datetime.now(timezone.utc).timestamp()
else:
self.build_id = build_id
self.buildid = build_id
def get_filtered_nodes(self, node_filter):
effective_nodes = self.get_baremetal_nodes()

62
tests/test_statemgmt.py Normal file
View File

@ -0,0 +1,62 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from helm_drydock.statemgmt import SiteDesign
import helm_drydock.model.site as site
import helm_drydock.model.network as network
import pytest
import shutil
import os
import helm_drydock.ingester.plugins.yaml
class TestClass(object):
def setup_method(self, method):
print("Running test {0}".format(method.__name__))
def test_sitedesign_merge(self):
design_data = SiteDesign()
initial_site = site.Site(**{'apiVersion': 'v1.0',
'metadata': {
'name': 'testsite',
},
})
net_a = network.Network(**{ 'apiVersion': 'v1.0',
'metadata': {
'name': 'net_a',
'region': 'testsite',
},
'spec': {
'cidr': '172.16.0.0/24',
}})
net_b = network.Network(**{ 'apiVersion': 'v1.0',
'metadata': {
'name': 'net_b',
'region': 'testsite',
},
'spec': {
'cidr': '172.16.0.1/24',
}})
design_data.add_site(initial_site)
design_data.add_network(net_a)
design_update = SiteDesign()
design_update.add_network(net_b)
design_data.merge_updates(design_update)
assert len(design_data.get_networks()) == 2