network: add retries for MTU set and team creation

Added more retries to check if the NICs
created by the teaming module are seen by the OS,
are up and running and MTU can be properly set
on them.

Change-Id: I92d547f8cf1165e0f9220c6e6ff92af013ee33d5

cloudbaseinit/osutils/windows.py

@@ -39,6 +39,7 @@ import winerror
 from cloudbaseinit import exception
 from cloudbaseinit.osutils import base
 from cloudbaseinit.utils import classloader
+from cloudbaseinit.utils import retry_decorator
 from cloudbaseinit.utils.windows import disk
 from cloudbaseinit.utils.windows import network
 from cloudbaseinit.utils.windows import privilege
@@ -757,6 +758,8 @@ class WindowsUtils(base.BaseOSUtils):
 
         return iface_index_list[0]["friendly_name"]
 
+    @retry_decorator.retry_decorator(
+        max_retry_count=3, exceptions=exception.ItemNotFoundException)
     def set_network_adapter_mtu(self, name, mtu):
         if not self.check_os_version(6, 0):
             raise exception.CloudbaseInitException(

cloudbaseinit/tests/utils/windows/test_netlbfo.py

@@ -42,15 +42,18 @@ class NetLBFOTest(unittest.TestCase):
         self._module_patcher.stop()
 
     @mock.patch(MODPATH + '.NetLBFOTeamManager._get_primary_adapter_name')
+    @mock.patch(MODPATH + '.NetLBFOTeamManager._create_team')
     @mock.patch(MODPATH + '.NetLBFOTeamManager._add_team_member')
     @mock.patch(MODPATH + '.NetLBFOTeamManager._set_primary_nic_vlan_id')
     @mock.patch(MODPATH + '.NetLBFOTeamManager._wait_for_nic')
     @mock.patch(MODPATH + '.NetLBFOTeamManager.delete_team')
     def _test_create_team(self, mock_delete_team, mock_wait_for_nic,
                           mock_set_primary_nic_vlan_id, mock_add_team_member,
-                          mock_get_primary_adapter_name, mode_not_found=False,
+                          mock_create_team, mock_get_primary_adapter_name,
-                          lb_algo_not_found=False, add_team_member_fail=False):
+                          mode_not_found=False, lb_algo_not_found=False,
+                          add_team_member_fail=False):
         mock_get_primary_adapter_name.return_value = mock.sentinel.pri_nic_name
+        mock_create_team.return_value = None
 
         lacp_timer = network_model.BOND_LACP_RATE_FAST
         members = [mock.sentinel.pri_nic_name, mock.sentinel.other_member]
@@ -58,6 +61,9 @@ class NetLBFOTest(unittest.TestCase):
         conn = self._wmi_mock.WMI.return_value
         mock_team = mock.Mock()
         conn.MSFT_NetLbfoTeam.new.return_value = mock_team
+        mock_team_nic = mock.Mock()
+        mock_team_nic.Name = mock.Mock()
+        conn.MSFT_NetLbfoTeamNic.return_value = [mock_team_nic]
 
         if mode_not_found:
             mode = "fake mode"
@@ -94,33 +100,17 @@ class NetLBFOTest(unittest.TestCase):
                 mock.sentinel.mac, mock.sentinel.pri_nic_name,
                 mock.sentinel.vlan_id, lacp_timer)
 
-        custom_options = [
-            {
-                u'name': u'TeamMembers',
-                u'value_type':
-                    self._mi_mock.MI_ARRAY | self._mi_mock.MI_STRING,
-                u'value': [mock.sentinel.pri_nic_name]
-            },
-            {
-                u'name': u'TeamNicName',
-                u'value_type': self._mi_mock.MI_STRING,
-                u'value': mock.sentinel.pri_nic_name
-            }
-        ]
-
-        operation_options = {u'custom_options': custom_options}
-        mock_team.put.assert_called_once_with(
-            operation_options=operation_options)
-
         mock_add_team_member.assert_called_once_with(
             conn, mock.sentinel.team_name, mock.sentinel.other_member)
 
         if not add_team_member_fail:
             mock_set_primary_nic_vlan_id.assert_called_once_with(
                 conn, mock.sentinel.team_name, mock.sentinel.vlan_id)
-
+            mock_create_team.assert_called_once_with(
+                conn, mock.sentinel.team_name, mock.sentinel.pri_nic_name,
+                2, 3, mock.sentinel.pri_nic_name, 1)
             mock_wait_for_nic.assert_called_once_with(
-                mock.sentinel.pri_nic_name)
+                mock_team_nic.Name)
         else:
             mock_delete_team.assert_called_once_with(mock.sentinel.team_name)
 
@@ -147,6 +137,41 @@ class NetLBFOTest(unittest.TestCase):
             name=mock.sentinel.team_name)
         mock_team.Delete_.assert_called_once_with()
 
+    def test_create_team_private(self):
+        conn = self._wmi_mock.WMI.return_value
+        mock_team = mock.Mock()
+        conn.MSFT_NetLbfoTeam.new.return_value = mock_team
+        teaming_mode = 1
+        lb_algo = 2
+        lacp_timer = 1
+
+        custom_options = [
+            {
+                u'name': u'TeamMembers',
+                u'value_type':
+                    self._mi_mock.MI_ARRAY | self._mi_mock.MI_STRING,
+                u'value': [mock.sentinel.private_nic_team]
+            },
+            {
+                u'name': u'TeamNicName',
+                u'value_type': self._mi_mock.MI_STRING,
+                u'value': mock.sentinel.team_nic_name
+            }
+        ]
+
+        operation_options = {u'custom_options': custom_options}
+        self._netlbfo.NetLBFOTeamManager()._create_team(
+            conn, mock.sentinel.team_name, mock.sentinel.team_nic_name,
+            teaming_mode, lb_algo, mock.sentinel.private_nic_team,
+            lacp_timer)
+
+        self.assertEqual(mock.sentinel.team_name, mock_team.Name)
+        self.assertEqual(teaming_mode, mock_team.TeamingMode)
+        self.assertEqual(lb_algo, mock_team.LoadBalancingAlgorithm)
+        self.assertEqual(lacp_timer, mock_team.LacpTimer)
+        mock_team.put.assert_called_once_with(
+            operation_options=operation_options)
+
     @mock.patch(MODPATH + '.NetLBFOTeamManager._wait_for_nic')
     def test_add_team_nic(self, mock_wait_for_nic):
         conn = self._wmi_mock.WMI.return_value

cloudbaseinit/utils/windows/netlbfo.py

@@ -13,7 +13,6 @@
 #    under the License.
 
 import sys
-import time
 
 import mi
 from oslo_log import log as oslo_logging
@@ -23,6 +22,7 @@ from cloudbaseinit import exception
 from cloudbaseinit.models import network as network_model
 from cloudbaseinit.osutils import factory as osutils_factory
 from cloudbaseinit.utils import network_team
+from cloudbaseinit.utils import retry_decorator
 
 LBFO_TEAM_MODE_STATIC = 0
 LBFO_TEAM_MODE_SWITCH_INDEPENDENT = 1
@@ -78,6 +78,7 @@ class NetLBFOTeamManager(network_team.BaseNetworkTeamManager):
         return primary_adapter_name
 
     @staticmethod
+    @retry_decorator.retry_decorator(max_retry_count=3)
     def _add_team_member(conn, team_name, member):
         team_member = conn.MSFT_NetLbfoTeamMember.new()
         team_member.Team = team_name
@@ -90,6 +91,7 @@ class NetLBFOTeamManager(network_team.BaseNetworkTeamManager):
         team_member.put(operation_options=operation_options)
 
     @staticmethod
+    @retry_decorator.retry_decorator(max_retry_count=3)
     def _set_primary_nic_vlan_id(conn, team_name, vlan_id):
         team_nic = conn.MSFT_NetLbfoTeamNIC(Team=team_name, Primary=True)[0]
 
@@ -102,6 +104,34 @@ class NetLBFOTeamManager(network_team.BaseNetworkTeamManager):
         operation_options = {u'custom_options': custom_options}
         team_nic.put(operation_options=operation_options)
 
+    @staticmethod
+    @retry_decorator.retry_decorator(max_retry_count=3)
+    def _create_team(conn, team_name, nic_name, teaming_mode, lb_algo,
+                     primary_adapter_name, lacp_timer=None):
+
+        team = conn.MSFT_NetLbfoTeam.new()
+        team.Name = team_name
+        team.TeamingMode = teaming_mode
+        team.LoadBalancingAlgorithm = lb_algo
+        if lacp_timer:
+            team.LacpTimer = lacp_timer
+
+        custom_options = [
+            {
+                u'name': u'TeamMembers',
+                u'value_type': mi.MI_ARRAY | mi.MI_STRING,
+                u'value': [primary_adapter_name]
+            },
+            {
+                u'name': u'TeamNicName',
+                u'value_type': mi.MI_STRING,
+                u'value': nic_name
+            }
+        ]
+
+        operation_options = {u'custom_options': custom_options}
+        team.put(operation_options=operation_options)
+
     def create_team(self, team_name, mode, load_balancing_algorithm,
                     members, mac_address, primary_nic_name=None,
                     primary_nic_vlan_id=None, lacp_timer=None):
@@ -124,30 +154,13 @@ class NetLBFOTeamManager(network_team.BaseNetworkTeamManager):
                 raise exception.ItemNotFoundException(
                     "Unsupported LB algorithm: %s" % load_balancing_algorithm)
 
-        team = conn.MSFT_NetLbfoTeam.new()
+        if lacp_timer is not None and teaming_mode == LBFO_TEAM_MODE_LACP:
-        team.Name = team_name
+            lacp_timer = NETWORK_MODEL_LACP_RATE_MAP[lacp_timer]
-        team.TeamingMode = teaming_mode
-        team.LoadBalancingAlgorithm = lb_algo
-
-        if lacp_timer is not None and team.TeamingMode == LBFO_TEAM_MODE_LACP:
-            team.LacpTimer = NETWORK_MODEL_LACP_RATE_MAP[lacp_timer]
 
         nic_name = primary_nic_name or team_name
-        custom_options = [
-            {
-                u'name': u'TeamMembers',
-                u'value_type': mi.MI_ARRAY | mi.MI_STRING,
-                u'value': [primary_adapter_name]
-            },
-            {
-                u'name': u'TeamNicName',
-                u'value_type': mi.MI_STRING,
-                u'value': nic_name
-            }
-        ]
 
-        operation_options = {u'custom_options': custom_options}
+        self._create_team(conn, team_name, nic_name, teaming_mode, lb_algo,
-        team.put(operation_options=operation_options)
+                          primary_adapter_name, lacp_timer)
 
         try:
             for member in members:
@@ -158,31 +171,21 @@ class NetLBFOTeamManager(network_team.BaseNetworkTeamManager):
                 self._set_primary_nic_vlan_id(
                     conn, team_name, primary_nic_vlan_id)
 
-            if primary_nic_name != team_name or primary_nic_vlan_id is None:
+            nic_name = conn.MSFT_NetLbfoTeamNic(team=team_name)[0].Name
-                # TODO(alexpilotti): query the new nic name
+            self._wait_for_nic(nic_name)
-                # When the nic name equals the bond name and a VLAN ID is set,
-                # the nick name is changed.
-                self._wait_for_nic(nic_name)
         except Exception as ex:
             self.delete_team(team_name)
             raise ex
 
     @staticmethod
+    @retry_decorator.retry_decorator(max_retry_count=10)
     def _wait_for_nic(nic_name):
         conn = wmi.WMI(moniker='//./root/cimv2')
-        max_count = 100
+        if not conn.Win32_NetworkAdapter(NetConnectionID=nic_name):
-        i = 0
+            raise exception.ItemNotFoundException(
-        while True:
+                "Cannot find NIC: %s" % nic_name)
-            if len(conn.Win32_NetworkAdapter(NetConnectionID=nic_name)):
-                break
-            else:
-                i += 1
-                if i >= max_count:
-                    raise exception.ItemNotFoundException(
-                        "Cannot find team NIC: %s" % nic_name)
-                LOG.debug("Waiting for team NIC: %s", nic_name)
-                time.sleep(1)
 
+    @retry_decorator.retry_decorator(max_retry_count=3)
     def add_team_nic(self, team_name, nic_name, vlan_id):
         conn = wmi.WMI(moniker='root/standardcimv2')
         team_nic = conn.MSFT_NetLbfoTeamNIC.new()
@@ -193,6 +196,7 @@ class NetLBFOTeamManager(network_team.BaseNetworkTeamManager):
         # Ensure that the NIC is visible in the OS before returning
         self._wait_for_nic(nic_name)
 
+    @retry_decorator.retry_decorator(max_retry_count=3)
     def delete_team(self, team_name):
         conn = wmi.WMI(moniker='root/standardcimv2')
         teams = conn.MSFT_NetLbfoTeam(name=team_name)