Merge "NSXvAdmin: Fix mismatches for security-groups"

vmware_nsx/plugins/nsx_v/plugin.py

@@ -2498,71 +2498,84 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
         if mapping is not None:
             return mapping['ip_section_id']
 
-    def create_security_group(self, context, security_group,
+    def _create_fw_section_for_security_group(self,
-                              default_sg=False):
+                                              context,
-        """Create a security group."""
+                                              securitygroup,
-        sg_data = security_group['security_group']
+                                              nsx_sg_id):
-        sg_data["id"] = str(uuid.uuid4())
+        logging = (cfg.CONF.nsxv.log_security_groups_allowed_traffic or
+                   securitygroup[sg_logging.LOGGING])
+        section_name = self.nsx_sg_utils.get_nsx_section_name(securitygroup)
+        nsx_rules = []
+        # Translate Neutron rules to NSXv fw rules and construct the fw section
+        for rule in securitygroup['security_group_rules']:
+            nsx_rule = self._create_nsx_rule(
+                context, rule, nsx_sg_id, logged=logging)
+            nsx_rules.append(nsx_rule)
+        section = self.nsx_sg_utils.get_section_with_rules(
+            section_name, nsx_rules)
+        # Execute REST API for creating the section
+        h, c = self.nsx_v.vcns.create_section(
+            'ip', self.nsx_sg_utils.to_xml_string(section),
+            insert_before=self.default_section)
+        rule_pairs = self.nsx_sg_utils.get_rule_id_pair_from_section(c)
+        # Add database associations for fw section and rules
+        nsxv_db.add_neutron_nsx_section_mapping(
+            context.session, securitygroup['id'], h['location'])
+        for pair in rule_pairs:
+            # Save nsx rule id in the DB for future access
+            nsxv_db.add_neutron_nsx_rule_mapping(
+                context.session, pair['neutron_id'], pair['nsx_id'])
 
-        new_security_group = super(NsxVPluginV2, self).create_security_group(
+    def _create_nsx_security_group(self, context, securitygroup):
-            context, security_group, default_sg)
+        nsx_sg_name = self.nsx_sg_utils.get_nsx_sg_name(securitygroup)
-        sg_id = new_security_group['id']
-
-        nsx_sg_name = self.nsx_sg_utils.get_nsx_sg_name(sg_data)
         # NSX security-group config
         sg_dict = {"securitygroup":
                    {"name": nsx_sg_name,
-                    "description": sg_data['description']}}
+                    "description": securitygroup['description']}}
-        # Translate Neutron rules to NSXv fw rules and construct the fw section
+        # Create the nsx security group
-        nsx_sg_id = section_uri = None
+        h, nsx_sg_id = self.nsx_v.vcns.create_security_group(sg_dict)
+
+        # Save moref in the DB for future access
+        nsx_db.add_neutron_nsx_security_group_mapping(
+            context.session, securitygroup['id'], nsx_sg_id)
+        return nsx_sg_id
+
+    def _process_security_group_create_backend_resources(self,
+                                                         context,
+                                                         securitygroup):
+        nsx_sg_id = self._create_nsx_security_group(context, securitygroup)
         try:
-            log_all_rules = cfg.CONF.nsxv.log_security_groups_allowed_traffic
+            self._create_fw_section_for_security_group(
-            # Create the nsx security group
+                context, securitygroup, nsx_sg_id)
-            h, nsx_sg_id = self.nsx_v.vcns.create_security_group(sg_dict)
-
-            section_name = self.nsx_sg_utils.get_nsx_section_name(nsx_sg_name)
-            logging = sg_data.get(sg_logging.LOGGING, False)
-            nsx_rules = []
-            for rule in new_security_group['security_group_rules']:
-                nsx_rule = self._create_nsx_rule(
-                    context, rule, nsx_sg_id, logged=log_all_rules or logging)
-                nsx_rules.append(nsx_rule)
-            section = self.nsx_sg_utils.get_section_with_rules(
-                section_name, nsx_rules)
-
-            # Execute REST API for creating the section
-            h, c = self.nsx_v.vcns.create_section(
-                'ip', self.nsx_sg_utils.to_xml_string(section),
-                insert_before=self.default_section)
-            section_uri = h['location']
-            rule_pairs = self.nsx_sg_utils.get_rule_id_pair_from_section(c)
-
-            # Save moref in the DB for future access
-            nsx_db.add_neutron_nsx_security_group_mapping(
-                context.session, sg_id, nsx_sg_id)
-            # Add database associations for fw section and rules
-            nsxv_db.add_neutron_nsx_section_mapping(
-                context.session, sg_id, section_uri)
-            self._process_security_group_properties_create(
-                context, new_security_group, sg_data)
-            for pair in rule_pairs:
-                # Save nsx rule id in the DB for future access
-                nsxv_db.add_neutron_nsx_rule_mapping(context.session,
-                                                     pair['neutron_id'],
-                                                     pair['nsx_id'])
-
-            # Add this Security Group to the Security Groups container
-            self._add_member_to_security_group(self.sg_container_id, nsx_sg_id)
         except Exception:
+            with excutils.save_and_reraise_exception():
+                self._delete_nsx_security_group(nsx_sg_id)
+
+        # Add this Security Group to the Security Groups container
+        self._add_member_to_security_group(self.sg_container_id, nsx_sg_id)
+
+    def create_security_group(self, context, security_group, default_sg=False):
+        """Create a security group."""
+        sg_data = security_group['security_group']
+        sg_id = sg_data["id"] = str(uuid.uuid4())
+
+        new_security_group = super(NsxVPluginV2, self).create_security_group(
+            context, security_group, default_sg)
+        self._process_security_group_properties_create(
+            context, new_security_group, sg_data)
+
+        try:
+            self._process_security_group_create_backend_resources(
+                context, new_security_group)
+        except Exception:
+            # Couldn't create backend resources, rolling back neutron db
+            # changes.
             with excutils.save_and_reraise_exception():
                 # Delete security-group and its associations from database,
                 # Only admin can delete the default security-group
                 if default_sg:
                     context = context.elevated()
                 super(NsxVPluginV2, self).delete_security_group(context, sg_id)
-                # Delete the created nsx security-group and the fw section
-                self._delete_section(section_uri)
-                self._delete_nsx_security_group(nsx_sg_id)
                 LOG.exception(_LE('Failed to create security group'))
         return new_security_group
 
@@ -2581,7 +2594,7 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
         section = self.nsx_sg_utils.parse_section(c)
         if set(['name', 'description']) & set(s.keys()):
             nsx_sg_name = self.nsx_sg_utils.get_nsx_sg_name(sg_data)
-            section_name = self.nsx_sg_utils.get_nsx_section_name(nsx_sg_name)
+            section_name = self.nsx_sg_utils.get_nsx_section_name(sg_data)
             self.nsx_v.vcns.update_security_group(
                 nsx_sg_id, nsx_sg_name, sg_data['description'])
             section.attrib['name'] = section_name

vmware_nsx/plugins/nsx_v/vshield/securitygroup_utils.py

@@ -141,8 +141,8 @@ class NsxSecurityGroupUtils(object):
     def get_nsx_sg_name(self, sg_data):
         return '%(name)s (%(id)s)' % sg_data
 
-    def get_nsx_section_name(self, nsx_sg_name):
+    def get_nsx_section_name(self, sg_data):
-        return 'SG Section: %s' % nsx_sg_name
+        return 'SG Section: %s' % self.get_nsx_sg_name(sg_data)
 
     def parse_and_get_section_id(self, section_xml):
         section = et.fromstring(section_xml)

vmware_nsx/shell/admin/plugins/nsxv/resources/securitygroups.py

@@ -18,10 +18,13 @@ import xml.etree.ElementTree as et
 
 from neutron.callbacks import registry
 from neutron import context
+from neutron.db import models_v2
 from neutron.db import securitygroups_db
 
+from vmware_nsx.db import db as nsx_db
 from vmware_nsx.db import nsx_models
 from vmware_nsx.db import nsxv_models
+from vmware_nsx import plugin
 from vmware_nsx.shell.admin.plugins.common import constants
 from vmware_nsx.shell.admin.plugins.common import formatters
 from vmware_nsx.shell.admin.plugins.common import utils as admin_utils
@@ -32,7 +35,13 @@ from vmware_nsx.shell import nsxadmin
 LOG = logging.getLogger(__name__)
 
 
-class NeutronSecurityGroupDB(utils.NeutronDbClient):
+class NsxVPluginWrapper(plugin.NsxVPlugin):
+    def _start_rpc_listeners(self):
+        pass
+
+
+class NeutronSecurityGroupDB(utils.NeutronDbClient,
+                             securitygroups_db.SecurityGroupDbMixin):
     def __init__(self):
         super(NeutronSecurityGroupDB, self)
         # FIXME(roeyc): context is already defined in NeutrondDbClient
@@ -48,15 +57,47 @@ class NeutronSecurityGroupDB(utils.NeutronDbClient):
                 nsx_models.NeutronNsxSecurityGroupMapping).all()
         sg_mappings = [{'name': mapp.name,
                         'id': mapp.id,
-                        'section-id': mapp.ip_section_id.split('/')[-1],
+                        'section-uri': mapp.ip_section_id,
                         'nsx-securitygroup-id': mapp.nsx_id}
                        for mapp in q]
         return sg_mappings
 
+    def get_security_group(self, sg_id):
+        return super(NeutronSecurityGroupDB, self).get_security_group(
+            self.context, sg_id)
+
     def get_security_groups(self):
         return super(NeutronSecurityGroupDB,
                      self).get_security_groups(self.context)
 
+    def delete_security_group_section_mapping(self, sg_id):
+        fw_mapping = self.context.session.query(
+            nsxv_models.NsxvSecurityGroupSectionMapping).filter_by(
+                neutron_id=sg_id).one_or_none()
+        if fw_mapping:
+            with self.context.session.begin(subtransactions=True):
+                self.context.session.delete(fw_mapping)
+
+    def delete_security_group_backend_mapping(self, sg_id):
+        sg_mapping = self.context.session.query(
+            nsx_models.NeutronNsxSecurityGroupMapping).filter_by(
+                neutron_id=sg_id).one_or_none()
+        if sg_mapping:
+            with self.context.session.begin(subtransactions=True):
+                self.context.session.delete(sg_mapping)
+
+    def get_vnics_in_security_group(self, security_group_id):
+        vnics = []
+        query = self.context.session.query(
+            models_v2.Port.id, models_v2.Port.device_id
+        ).join(securitygroups_db.SecurityGroupPortBinding).filter_by(
+            security_group_id=security_group_id).all()
+        for p in query:
+            vnic_index = plugin._get_port_vnic_index(self.context, p.id)
+            vnic_id = plugin._get_port_vnic_id(vnic_index, p.device_id)
+            vnics.append(vnic_id)
+        return vnics
+
 
 class NsxFirewallAPI(object):
     def __init__(self):
@@ -91,6 +132,7 @@ class NsxFirewallAPI(object):
 
 neutron_sg = NeutronSecurityGroupDB()
 nsxv_firewall = NsxFirewallAPI()
+plugin = NsxVPluginWrapper()
 
 
 def _log_info(resource, data, attrs=['name', 'id']):
@@ -113,13 +155,21 @@ def list_mismatches_handler(resource):
     return wrap
 
 
+def fix_mismatches_handler(resource):
+    def wrap(func):
+        registry.subscribe(func, resource,
+                           nsxadmin.Operations.FIX_MISMATCH.value)
+        return func
+    return wrap
+
+
 @list_handler(constants.SECURITY_GROUPS)
 @admin_utils.output_header
 def neutron_list_security_groups_mappings(resource, event, trigger, **kwargs):
     sg_mappings = neutron_sg.get_security_groups_mappings()
     _log_info(constants.SECURITY_GROUPS,
               sg_mappings,
-              attrs=['name', 'id', 'section-id', 'nsx-securitygroup-id'])
+              attrs=['name', 'id', 'section-uri', 'nsx-securitygroup-id'])
     return bool(sg_mappings)
 
 
@@ -139,41 +189,87 @@ def nsx_list_security_groups(resource, event, trigger, **kwargs):
     return bool(nsx_secgroups)
 
 
-@list_mismatches_handler(constants.FIREWALL_NSX_GROUPS)
+def _find_missing_security_groups():
-@admin_utils.output_header
-def list_missing_security_groups(resource, event, trigger, **kwargs):
     nsx_secgroups = nsxv_firewall.list_security_groups()
     sg_mappings = neutron_sg.get_security_groups_mappings()
-    missing_nsx_secgroups = []
+    missing_secgroups = {}
     for sg_db in sg_mappings:
         for nsx_sg in nsx_secgroups:
             if nsx_sg['id'] == sg_db['nsx-securitygroup-id']:
                 break
         else:
-            missing_nsx_secgroups.append({'securitygroup-name': sg_db['name'],
+            missing_secgroups[sg_db['id']] = sg_db
-                                          'securitygroup-id': sg_db['id'],
+    return missing_secgroups
-                                          'nsx-securitygroup-id':
+
-                                          sg_db['nsx-securitygroup-id']})
+
-    _log_info(constants.FIREWALL_NSX_GROUPS, missing_nsx_secgroups,
+@list_mismatches_handler(constants.FIREWALL_NSX_GROUPS)
+@admin_utils.output_header
+def list_missing_security_groups(resource, event, trigger, **kwargs):
+    sgs_with_missing_nsx_group = _find_missing_security_groups()
+    missing_securitgroups_info = [
+        {'securitygroup-name': sg['name'],
+         'securitygroup-id': sg['id'],
+         'nsx-securitygroup-id':
+         sg['nsx-securitygroup-id']}
+        for sg in sgs_with_missing_nsx_group.values()]
+    _log_info(constants.FIREWALL_NSX_GROUPS, missing_securitgroups_info,
               attrs=['securitygroup-name', 'securitygroup-id',
                      'nsx-securitygroup-id'])
-    return bool(missing_nsx_secgroups)
+    return bool(missing_securitgroups_info)
+
+
+def _find_missing_sections():
+    fw_sections = nsxv_firewall.list_fw_sections()
+    sg_mappings = neutron_sg.get_security_groups_mappings()
+    missing_sections = {}
+    for sg_db in sg_mappings:
+        for fw_section in fw_sections:
+            if fw_section['id'] == sg_db.get('section-uri', '').split('/')[-1]:
+                break
+        else:
+            missing_sections[sg_db['id']] = sg_db
+    return missing_sections
 
 
 @list_mismatches_handler(constants.FIREWALL_SECTIONS)
 @admin_utils.output_header
 def list_missing_firewall_sections(resource, event, trigger, **kwargs):
-    fw_sections = nsxv_firewall.list_fw_sections()
+    sgs_with_missing_section = _find_missing_sections()
-    sg_mappings = neutron_sg.get_security_groups_mappings()
+    missing_sections_info = [{'securitygroup-name': sg['name'],
-    missing_sections = []
+                              'securitygroup-id': sg['id'],
-    for sg_db in sg_mappings:
+                              'section-id': sg['section-uri']}
-        for fw_section in fw_sections:
+                             for sg in sgs_with_missing_section.values()]
-            if fw_section['id'] == sg_db['section-id']:
+    _log_info(constants.FIREWALL_SECTIONS, missing_sections_info,
-                break
+              attrs=['securitygroup-name', 'securitygroup-id', 'section-uri'])
-        else:
+    return bool(missing_sections_info)
-            missing_sections.append({'securitygroup-name': sg_db['name'],
+
-                                     'securitygroup-id': sg_db['id'],
+
-                                     'section-id': sg_db['section-id']})
+@fix_mismatches_handler(constants.SECURITY_GROUPS)
-    _log_info(constants.FIREWALL_SECTIONS, missing_sections,
+@admin_utils.output_header
-              attrs=['securitygroup-name', 'securitygroup-id', 'section-id'])
+def fix_security_groups(resource, event, trigger, **kwargs):
-    return bool(missing_sections)
+    context_ = context.get_admin_context()
+    sgs_with_missing_section = _find_missing_sections()
+    sgs_with_missing_nsx_group = _find_missing_security_groups()
+    plugin = NsxVPluginWrapper()
+    # If only the fw section is missing then create it.
+    for sg_id in (set(sgs_with_missing_section.keys()) -
+                  set(sgs_with_missing_nsx_group.keys())):
+        neutron_sg.delete_security_group_section_mapping(sg_id)
+        secgroup = plugin.get_security_group(context_, sg_id)
+        plugin._create_fw_section_for_security_group(
+            context_, secgroup,
+            sgs_with_missing_section[sg_id]['nsx-securitygroup-id'])
+
+    # If nsx security-group is missing then create both nsx security-group and
+    # a new fw section (remove old one).
+    for sg_id, sg in sgs_with_missing_nsx_group.items():
+        secgroup = plugin.get_security_group(context_, sg_id)
+        if sg_id not in sgs_with_missing_section:
+            plugin._delete_section(sg['section-uri'])
+        neutron_sg.delete_security_group_section_mapping(sg_id)
+        neutron_sg.delete_security_group_backend_mapping(sg_id)
+        plugin._process_security_group_create_backend_resources(context_,
+                                                                secgroup)
+        nsx_id = nsx_db.get_nsx_security_group_id(context_.session, sg_id)
+        for vnic_id in neutron_sg.get_vnics_in_security_group(sg_id):
+            plugin._add_member_to_security_group(nsx_id, vnic_id)

vmware_nsx/shell/nsxadmin.py

@@ -115,7 +115,8 @@ nsxv_resources = {
                                  [Operations.LIST.value,
                                   Operations.NSX_UPDATE.value]),
     constants.SECURITY_GROUPS: Resource(constants.SECURITY_GROUPS,
-                                        [Operations.LIST.value]),
+                                        [Operations.LIST.value,
+                                         Operations.FIX_MISMATCH.value]),
     constants.FIREWALL_SECTIONS: Resource(constants.FIREWALL_SECTIONS,
                                           [Operations.LIST.value,
                                            Operations.LIST_MISMATCHES.value]),