From dac109662ed54683101d2ba316c56f1d20b8c258 Mon Sep 17 00:00:00 2001 From: Adit Sarfaty Date: Tue, 31 Jul 2018 10:40:23 +0300 Subject: [PATCH] NSX|V+V3: Move FW section logging update to admin utility On the plugin init there is a side process going over all the security group rules in the NSX DFW checking if their logging flag should be updated according to the global configuration flag. Since this is relevant only in case the global config flag log_security_groups_allowed_traffic was updated by the user, which is very rare, this patch removed it from the code, and replaced it with an admin utility that can be used. This will make the plugin initialization process quicker and prevent unnecessary load on the NSX. Change-Id: I233915e589b53ccb4b76a3ef3d24bb56c0459e92 --- doc/source/admin_util.rst | 8 +++ vmware_nsx/plugins/nsx_v/plugin.py | 42 -------------- vmware_nsx/plugins/nsx_v3/plugin.py | 22 ------- .../plugins/nsxv/resources/securitygroups.py | 58 +++++++++++++++++++ .../admin/plugins/nsxv/resources/utils.py | 3 - .../plugins/nsxv3/resources/securitygroups.py | 41 +++++++++++++ .../admin/plugins/nsxv3/resources/utils.py | 3 - vmware_nsx/shell/resources.py | 7 ++- vmware_nsx/tests/unit/nsx_v/test_plugin.py | 8 --- vmware_nsx/tests/unit/nsx_v3/test_plugin.py | 4 -- .../tests/unit/shell/test_admin_utils.py | 6 +- 11 files changed, 116 insertions(+), 86 deletions(-) diff --git a/doc/source/admin_util.rst b/doc/source/admin_util.rst index 59cb66fa48..64c17e5580 100644 --- a/doc/source/admin_util.rst +++ b/doc/source/admin_util.rst @@ -269,6 +269,10 @@ Security Groups, Firewall and Spoofguard nsxadmin -r security-groups -o migrate-to-policy --property policy-id=policy-10 --property security-group-id=733f0741-fa2c-4b32-811c-b78e4dc8ec39 +- Update logging flag of the security groups on the NSX DFW + + nsxadmin -r security-groups -o update-logging --property log-allowed-traffic=true + - Spoofguard support:: nsxadmin -r spoofguard-policy -o list-mismatches @@ -390,6 +394,10 @@ Security Groups & NSX Security Groups nsxadmin -r nsx-security-groups -o migrate-to-dynamic-criteria +- Update logging flag of the security groups on the NSX DFW + + nsxadmin -r security-groups -o update-logging --property log-allowed-traffic=true + Firewall Sections ~~~~~~~~~~~~~~~~~ diff --git a/vmware_nsx/plugins/nsx_v/plugin.py b/vmware_nsx/plugins/nsx_v/plugin.py index 05d08d26e2..17399c3233 100644 --- a/vmware_nsx/plugins/nsx_v/plugin.py +++ b/vmware_nsx/plugins/nsx_v/plugin.py @@ -295,7 +295,6 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin, "NSX 6.3 onwards") self.sg_container_id = self._create_security_group_container() self.default_section = self._create_cluster_default_fw_section() - self._process_security_groups_rules_logging() self._router_managers = managers.RouterTypeManager(self) @@ -574,47 +573,6 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin, section_id = self.nsx_sg_utils.parse_and_get_section_id(c) return section_id - def _process_security_groups_rules_logging(self): - - def process_security_groups_rules_logging(*args, **kwargs): - with locking.LockManager.get_lock('nsx-dfw-section', - lock_file_prefix='dfw-section'): - context = n_context.get_admin_context() - log_allowed = cfg.CONF.nsxv.log_security_groups_allowed_traffic - - # If the section/sg is already logged, then no action is - # required. - for sg in [sg for sg in self.get_security_groups(context) - if sg.get(sg_logging.LOGGING) is False]: - if sg.get(sg_policy.POLICY): - # Logging is not relevant with a policy - continue - - section_uri = self._get_section_uri(context.session, - sg['id']) - if section_uri is None: - continue - - # Section/sg is not logged, update rules logging according - # to the 'log_security_groups_allowed_traffic' config - # option. - try: - h, c = self.nsx_v.vcns.get_section(section_uri) - section = self.nsx_sg_utils.parse_section(c) - section_needs_update = ( - self.nsx_sg_utils.set_rules_logged_option( - section, log_allowed)) - if section_needs_update: - self.nsx_v.vcns.update_section( - section_uri, - self.nsx_sg_utils.to_xml_string(section), h) - except Exception as exc: - LOG.error('Unable to update security group %(sg)s ' - 'section for logging. %(e)s', - {'e': exc, 'sg': sg['id']}) - - c_utils.spawn_n(process_security_groups_rules_logging) - def _create_dhcp_static_binding(self, context, neutron_port_db): network_id = neutron_port_db['network_id'] diff --git a/vmware_nsx/plugins/nsx_v3/plugin.py b/vmware_nsx/plugins/nsx_v3/plugin.py index b7e861f940..a970cd38e3 100644 --- a/vmware_nsx/plugins/nsx_v3/plugin.py +++ b/vmware_nsx/plugins/nsx_v3/plugin.py @@ -274,7 +274,6 @@ class NsxV3Plugin(agentschedulers_db.AZDhcpAgentSchedulerDbMixin, self._init_dhcp_metadata() self._prepare_default_rules() - self._process_security_group_logging() # init profiles on nsx backend self._init_nsx_profiles() @@ -779,27 +778,6 @@ class NsxV3Plugin(agentschedulers_db.AZDhcpAgentSchedulerDbMixin, tags=self.nsxlib.build_v3_api_version_tag()) return self._get_port_security_profile() - def _process_security_group_logging(self): - def process_security_group_logging(*args, **kwargs): - context = q_context.get_admin_context() - log_all_rules = cfg.CONF.nsx_v3.log_security_groups_allowed_traffic - secgroups = self.get_security_groups(context, - fields=['id', - sg_logging.LOGGING]) - for sg in [sg for sg in secgroups - if sg.get(sg_logging.LOGGING) is False]: - nsgroup_id, section_id = nsx_db.get_sg_mappings( - context.session, sg['id']) - if section_id: - try: - self.nsxlib.firewall_section.set_rule_logging( - section_id, logging=log_all_rules) - except nsx_lib_exc.ManagerError: - LOG.error("Failed to update firewall rule logging " - "for rule in section %s", section_id) - - utils.spawn_n(process_security_group_logging) - def _init_default_section_rules(self): with locking.LockManager.get_lock('nsxv3_default_section'): section_description = ("This section is handled by OpenStack to " diff --git a/vmware_nsx/shell/admin/plugins/nsxv/resources/securitygroups.py b/vmware_nsx/shell/admin/plugins/nsxv/resources/securitygroups.py index 62d1077709..247f68c8c0 100644 --- a/vmware_nsx/shell/admin/plugins/nsxv/resources/securitygroups.py +++ b/vmware_nsx/shell/admin/plugins/nsxv/resources/securitygroups.py @@ -31,6 +31,7 @@ from vmware_nsx.db import extended_security_group_rule as extend_sg_rule from vmware_nsx.db import nsx_models from vmware_nsx.db import nsxv_db from vmware_nsx.db import nsxv_models +from vmware_nsx.extensions import securitygrouplogging as sg_logging from vmware_nsx.extensions import securitygrouppolicy as sg_policy from vmware_nsx.shell.admin.plugins.common import constants from vmware_nsx.shell.admin.plugins.common import formatters @@ -464,6 +465,63 @@ def firewall_update_cluster_default_fw_section(resource, event, trigger, LOG.info("Cluster default FW section updated.") +@admin_utils.output_header +def update_security_groups_logging(resource, event, trigger, **kwargs): + """Update allowed traffic logging for all neutron security group rules""" + errmsg = ("Need to specify log-allowed-traffic property. Add --property " + "log-allowed-traffic=true/false") + if not kwargs.get('property'): + LOG.error("%s", errmsg) + return + properties = admin_utils.parse_multi_keyval_opt(kwargs['property']) + log_allowed_str = properties.get('log-allowed-traffic') + if not log_allowed_str or log_allowed_str.lower() not in ['true', 'false']: + LOG.error("%s", errmsg) + return + log_allowed = log_allowed_str.lower() == 'true' + + context = n_context.get_admin_context() + + with utils.NsxVPluginWrapper() as plugin: + vcns = plugin.nsx_v.vcns + sg_utils = plugin. nsx_sg_utils + # If the section/sg is already logged, then no action is + # required. + security_groups = plugin.get_security_groups(context) + LOG.info("Going to update logging of %s sections", + len(security_groups)) + for sg in [sg for sg in plugin.get_security_groups(context) + if sg.get(sg_logging.LOGGING) is False]: + if sg.get(sg_policy.POLICY): + # Logging is not relevant with a policy + continue + + section_uri = plugin._get_section_uri(context.session, + sg['id']) + if section_uri is None: + continue + + # Section/sg is not logged, update rules logging according + # to the 'log_security_groups_allowed_traffic' config + # option. + try: + h, c = vcns.get_section(section_uri) + section = sg_utils.parse_section(c) + section_needs_update = sg_utils.set_rules_logged_option( + section, log_allowed) + if section_needs_update: + vcns.update_section(section_uri, + sg_utils.to_xml_string(section), h) + except Exception as exc: + LOG.error('Unable to update security group %(sg)s ' + 'section for logging. %(e)s', + {'e': exc, 'sg': sg['id']}) + + +registry.subscribe(update_security_groups_logging, + constants.SECURITY_GROUPS, + shell.Operations.UPDATE_LOGGING.value) + registry.subscribe(migrate_sg_to_policy, constants.SECURITY_GROUPS, shell.Operations.MIGRATE_TO_POLICY.value) diff --git a/vmware_nsx/shell/admin/plugins/nsxv/resources/utils.py b/vmware_nsx/shell/admin/plugins/nsxv/resources/utils.py index 6185c25095..b0f3ec840a 100644 --- a/vmware_nsx/shell/admin/plugins/nsxv/resources/utils.py +++ b/vmware_nsx/shell/admin/plugins/nsxv/resources/utils.py @@ -75,9 +75,6 @@ class NsxVPluginWrapper(plugin.NsxVPlugin): # skip getting the Qos policy ID because get_object calls # plugin init again on admin-util environment - def _process_security_groups_rules_logging(self): - pass - def count_spawn_jobs(self): # check if there are any spawn jobs running return self.edge_manager._get_worker_pool().running() diff --git a/vmware_nsx/shell/admin/plugins/nsxv3/resources/securitygroups.py b/vmware_nsx/shell/admin/plugins/nsxv3/resources/securitygroups.py index e11f74a735..4762a787a8 100644 --- a/vmware_nsx/shell/admin/plugins/nsxv3/resources/securitygroups.py +++ b/vmware_nsx/shell/admin/plugins/nsxv3/resources/securitygroups.py @@ -29,6 +29,7 @@ from vmware_nsx.shell.admin.plugins.common import formatters from vmware_nsx.shell.admin.plugins.common import utils as admin_utils from vmware_nsx.shell.admin.plugins.nsxv3.resources import utils as v3_utils from vmware_nsx.shell import resources as shell +from vmware_nsxlib.v3 import exceptions as nsx_lib_exc from vmware_nsxlib.v3 import nsx_constants as consts from vmware_nsxlib.v3 import security @@ -314,6 +315,46 @@ def clean_orphaned_sections(resource, event, trigger, **kwargs): LOG.info("Backend firewall section %s was deleted.", sec['id']) +def update_security_groups_logging(resource, event, trigger, **kwargs): + """Update allowed traffic logging for all neutron security group rules""" + errmsg = ("Need to specify log-allowed-traffic property. Add --property " + "log-allowed-traffic=true/false") + if not kwargs.get('property'): + LOG.error("%s", errmsg) + return + properties = admin_utils.parse_multi_keyval_opt(kwargs['property']) + log_allowed_str = properties.get('log-allowed-traffic') + if not log_allowed_str or log_allowed_str.lower() not in ['true', 'false']: + LOG.error("%s", errmsg) + return + log_allowed = log_allowed_str.lower() == 'true' + + context = neutron_context.get_admin_context() + nsxlib = v3_utils.get_connected_nsxlib() + + with v3_utils.NsxV3PluginWrapper() as plugin: + secgroups = plugin.get_security_groups(context, + fields=['id', + sg_logging.LOGGING]) + LOG.info("Going to update logging of %s sections", + len(secgroups)) + for sg in [sg for sg in secgroups + if sg.get(sg_logging.LOGGING) is False]: + nsgroup_id, section_id = nsx_db.get_sg_mappings( + context.session, sg['id']) + if section_id: + try: + nsxlib.firewall_section.set_rule_logging( + section_id, logging=log_allowed) + except nsx_lib_exc.ManagerError: + LOG.error("Failed to update firewall rule logging " + "for rule in section %s", section_id) + + +registry.subscribe(update_security_groups_logging, + constants.SECURITY_GROUPS, + shell.Operations.UPDATE_LOGGING.value) + registry.subscribe(migrate_nsgroups_to_dynamic_criteria, constants.FIREWALL_NSX_GROUPS, shell.Operations.MIGRATE_TO_DYNAMIC_CRITERIA.value) diff --git a/vmware_nsx/shell/admin/plugins/nsxv3/resources/utils.py b/vmware_nsx/shell/admin/plugins/nsxv3/resources/utils.py index 59057890ee..d27f5be025 100644 --- a/vmware_nsx/shell/admin/plugins/nsxv3/resources/utils.py +++ b/vmware_nsx/shell/admin/plugins/nsxv3/resources/utils.py @@ -169,9 +169,6 @@ class NsxV3PluginWrapper(plugin.NsxV3Plugin): def _init_dhcp_metadata(self): pass - def _process_security_group_logging(self): - pass - def _extend_get_network_dict_provider(self, context, net): self._extend_network_dict_provider(context, net) # skip getting the Qos policy ID because get_object calls diff --git a/vmware_nsx/shell/resources.py b/vmware_nsx/shell/resources.py index e9bcc65776..efefb1adde 100644 --- a/vmware_nsx/shell/resources.py +++ b/vmware_nsx/shell/resources.py @@ -59,6 +59,7 @@ class Operations(enum.Enum): MIGRATE_TO_DYNAMIC_CRITERIA = 'migrate-to-dynamic-criteria' NSX_MIGRATE_V_V3 = 'nsx-migrate-v-v3' MIGRATE_TO_POLICY = 'migrate-to-policy' + UPDATE_LOGGING = 'update-logging' NSX_MIGRATE_EXCLUDE_PORTS = 'migrate-exclude-ports' MIGRATE_VDR_DHCP = 'migrate-vdr-dhcp' STATUS = 'status' @@ -80,7 +81,8 @@ class Resource(object): nsxv3_resources = { constants.SECURITY_GROUPS: Resource(constants.SECURITY_GROUPS, [Operations.LIST.value, - Operations.FIX_MISMATCH.value]), + Operations.FIX_MISMATCH.value, + Operations.UPDATE_LOGGING.value]), constants.FIREWALL_SECTIONS: Resource(constants.FIREWALL_SECTIONS, [Operations.LIST.value, Operations.LIST_MISMATCHES.value]), @@ -186,7 +188,8 @@ nsxv_resources = { constants.SECURITY_GROUPS: Resource(constants.SECURITY_GROUPS, [Operations.LIST.value, Operations.FIX_MISMATCH.value, - Operations.MIGRATE_TO_POLICY.value]), + Operations.MIGRATE_TO_POLICY.value, + Operations.UPDATE_LOGGING.value]), constants.FIREWALL_NSX_GROUPS: Resource( constants.FIREWALL_NSX_GROUPS, [Operations.LIST.value, Operations.LIST_MISMATCHES.value]), diff --git a/vmware_nsx/tests/unit/nsx_v/test_plugin.py b/vmware_nsx/tests/unit/nsx_v/test_plugin.py index ead0f7473f..b2610c41e7 100644 --- a/vmware_nsx/tests/unit/nsx_v/test_plugin.py +++ b/vmware_nsx/tests/unit/nsx_v/test_plugin.py @@ -214,10 +214,6 @@ class NsxVPluginV2TestCase(test_plugin.NeutronDbPluginV2TestCase): mock_deploy_backup_edges_at_backend = mock.patch("%s.%s" % ( vmware.EDGE_MANAGE_NAME, '_deploy_backup_edges_at_backend')) mock_deploy_backup_edges_at_backend.start() - mock_process_security_group_logging = mock.patch( - 'vmware_nsx.plugin.NsxVPlugin.' - '_process_security_groups_rules_logging') - mock_process_security_group_logging.start() self.default_res_pool = 'respool-28' cfg.CONF.set_override("resource_pool_id", self.default_res_pool, @@ -3988,10 +3984,6 @@ class NsxVSecurityGroupsTestCase(ext_sg.SecurityGroupDBTestCase): mock_check_backup_edge_pools = mock.patch("%s.%s" % ( vmware.EDGE_MANAGE_NAME, '_check_backup_edge_pools')) mock_check_backup_edge_pools.start() - mock_process_security_group_logging = mock.patch( - 'vmware_nsx.plugin.NsxVPlugin.' - '_process_security_groups_rules_logging') - mock_process_security_group_logging.start() c_utils.spawn_n = mock.Mock(side_effect=lambda f: f()) super(NsxVSecurityGroupsTestCase, self).setUp(plugin=plugin, diff --git a/vmware_nsx/tests/unit/nsx_v3/test_plugin.py b/vmware_nsx/tests/unit/nsx_v3/test_plugin.py index fac698b6e0..da760e40ce 100644 --- a/vmware_nsx/tests/unit/nsx_v3/test_plugin.py +++ b/vmware_nsx/tests/unit/nsx_v3/test_plugin.py @@ -197,10 +197,6 @@ class NsxV3PluginTestCaseMixin(test_plugin.NeutronDbPluginV2TestCase, 'neutron.scheduler.dhcp_agent_scheduler.AZAwareWeightScheduler') def mock_plugin_methods(self): - # mock unnecessary call which causes spawn - mock_process_security_group_logging = mock.patch.object( - nsx_plugin.NsxV3Plugin, '_process_security_group_logging') - mock_process_security_group_logging.start() # need to mock the global placeholder. This is due to the fact that # the generic security group tests assume that there is just one # security group. diff --git a/vmware_nsx/tests/unit/shell/test_admin_utils.py b/vmware_nsx/tests/unit/shell/test_admin_utils.py index 5d0a68ab57..6f6c1351a4 100644 --- a/vmware_nsx/tests/unit/shell/test_admin_utils.py +++ b/vmware_nsx/tests/unit/shell/test_admin_utils.py @@ -220,7 +220,8 @@ class TestNsxvAdminUtils(AbstractTestAdminUtils, "security-group-id=sg-1", "dvs-id=dvs-1", "moref=virtualwire-1", - "teamingpolicy=LACP_ACTIVE" + "teamingpolicy=LACP_ACTIVE", + "log-allowed-traffic=true" ] self._test_resources_with_args( resources.nsxv_resources, args) @@ -281,7 +282,8 @@ class TestNsxv3AdminUtils(AbstractTestAdminUtils, "metadata_proxy_uuid=e5b9b249-0034-4729-8ab6-fe4dacaa3a12", "nsx-id=e5b9b249-0034-4729-8ab6-fe4dacaa3a12", "availability-zone=default", - "server-ip=1.1.1.1" + "server-ip=1.1.1.1", + "log-allowed-traffic=true" ] # Create some neutron objects for the utilities to run on self._create_router()