diff --git a/bareon/actions/partitioning.py b/bareon/actions/partitioning.py index 954b39e..ee8acd9 100644 --- a/bareon/actions/partitioning.py +++ b/bareon/actions/partitioning.py @@ -21,6 +21,7 @@ from bareon.actions import base from bareon import errors from bareon.openstack.common import log as logging from bareon.utils import fs as fu +from bareon.utils import hardware as hu from bareon.utils import lvm as lu from bareon.utils import md as mu from bareon.utils import partition as pu @@ -53,6 +54,24 @@ opts = [ help='Allow to skip MD containers (fake raid leftovers) while ' 'cleaning the rest of MDs', ), + cfg.IntOpt( + 'partition_udev_settle_attempts', + default=10, + help='How many times udev settle will be called after partitioning' + ), + cfg.ListOpt( + 'multipath_lvm_filter', + default=['r|/dev/mapper/.*-part.*|', + 'r|/dev/dm-.*|', + 'r|/dev/disk/by-id/.*|'], + help='Extra filters for lvm.conf to force lvm work with partions ' + 'on multipath devices using /dev/mapper/-part links' + ), + cfg.StrOpt( + 'lvm_conf_path', + default='/etc/lvm/lvm.conf', + help='Path to LVM configuration file' + ) ] CONF = cfg.CONF @@ -95,6 +114,25 @@ class PartitioningAction(base.BaseAction): if not fs.keep_data and not found_images: fu.make_fs(fs.type, fs.options, fs.label, fs.device) + @staticmethod + def _make_partitions(parteds, wait_for_udev_settle=False): + for parted in parteds: + pu.make_label(parted.name, parted.label) + for prt in parted.partitions: + pu.make_partition(prt.device, prt.begin, prt.end, prt.type) + if wait_for_udev_settle: + utils.wait_for_udev_settle( + attempts=CONF.partition_udev_settle_attempts) + for flag in prt.flags: + pu.set_partition_flag(prt.device, prt.count, flag) + if prt.guid: + pu.set_gpt_type(prt.device, prt.count, prt.guid) + # If any partition to be created doesn't exist it's an error. + # Probably it's again 'device or resource busy' issue. + if not os.path.exists(prt.name): + raise errors.PartitionNotFoundError( + 'Partition %s not found after creation' % prt.name) + def _do_partitioning(self): LOG.debug('--- Partitioning disks (do_partitioning) ---') @@ -106,12 +144,6 @@ class PartitioningAction(base.BaseAction): lu.vgremove_all() lu.pvremove_all() - LOG.debug("Enabling udev's rules blacklisting") - utils.blacklist_udev_rules(udev_rules_dir=CONF.udev_rules_dir, - udev_rules_lib_dir=CONF.udev_rules_lib_dir, - udev_rename_substr=CONF.udev_rename_substr, - udev_empty_rule=CONF.udev_empty_rule) - for parted in self.driver.partition_scheme.parteds: for prt in parted.partitions: # We wipe out the beginning of every new partition @@ -130,25 +162,27 @@ class PartitioningAction(base.BaseAction): 'seek=%s' % max(prt.end - 3, 0), 'count=5', 'of=%s' % prt.device, check_exit_code=[0, 1]) + parteds = [] + parteds_with_rules = [] for parted in self.driver.partition_scheme.parteds: - pu.make_label(parted.name, parted.label) - for prt in parted.partitions: - pu.make_partition(prt.device, prt.begin, prt.end, prt.type) - for flag in prt.flags: - pu.set_partition_flag(prt.device, prt.count, flag) - if prt.guid: - pu.set_gpt_type(prt.device, prt.count, prt.guid) - # If any partition to be created doesn't exist it's an error. - # Probably it's again 'device or resource busy' issue. - if not os.path.exists(prt.name): - raise errors.PartitionNotFoundError( - 'Partition %s not found after creation' % prt.name) + if hu.is_multipath_device(parted.name): + parteds_with_rules.append(parted) + else: + parteds.append(parted) + + utils.blacklist_udev_rules(udev_rules_dir=CONF.udev_rules_dir, + udev_rules_lib_dir=CONF.udev_rules_lib_dir, + udev_rename_substr=CONF.udev_rename_substr, + udev_empty_rule=CONF.udev_empty_rule) + + self._make_partitions(parteds) - LOG.debug("Disabling udev's rules blacklisting") utils.unblacklist_udev_rules( udev_rules_dir=CONF.udev_rules_dir, udev_rename_substr=CONF.udev_rename_substr) + self._make_partitions(parteds_with_rules, wait_for_udev_settle=True) + # If one creates partitions with the same boundaries as last time, # there might be md and lvm metadata on those partitions. To prevent # failing of creating md and lvm devices we need to make sure diff --git a/bareon/drivers/deploy/nailgun.py b/bareon/drivers/deploy/nailgun.py index f92e51c..6b54b80 100644 --- a/bareon/drivers/deploy/nailgun.py +++ b/bareon/drivers/deploy/nailgun.py @@ -490,6 +490,8 @@ class Manager(BaseDeployDriver): bu.dump_runtime_uuid(bs_scheme.uuid, os.path.join(chroot, 'etc/nailgun-agent/config.yaml')) + bu.append_lvm_devices_filter(chroot, CONF.multipath_lvm_filter, + CONF.lvm_conf_path) bu.do_post_inst(chroot, allow_unsigned_file=CONF.allow_unsigned_file, force_ipv4_file=CONF.force_ipv4_file) diff --git a/bareon/objects/partition/parted.py b/bareon/objects/partition/parted.py index 7ec3315..ba30fe3 100644 --- a/bareon/objects/partition/parted.py +++ b/bareon/objects/partition/parted.py @@ -97,10 +97,14 @@ class Parted(base.Serializable): def next_name(self): if self.next_type() == 'extended': return None - separator = '' + special_devices = ('cciss', 'nvme', 'loop', 'md') if any(n in self.name for n in special_devices): separator = 'p' + elif '/dev/mapper' in self.name: + separator = '-part' + else: + separator = '' return '%s%s%s' % (self.name, separator, self.next_count()) def partition_by_name(self, name): diff --git a/bareon/tests/test_build_utils.py b/bareon/tests/test_build_utils.py index 25376bb..74d24b0 100644 --- a/bareon/tests/test_build_utils.py +++ b/bareon/tests/test_build_utils.py @@ -759,3 +759,107 @@ class BuildUtilsTestCase(unittest2.TestCase): '/test/dst/dir/mnt/dst/.mksquashfs.tmp.fake_uuid', 'myname' ) + + @mock.patch.object(utils, 'execute') + def test_get_config_value(self, mock_exec): + mock_exec.return_value = [r'foo=42', ''] + self.assertEqual(42, bu.get_lvm_config_value('fake_chroot', + 'section', 'foo')) + + mock_exec.return_value = [r'bar=0.5', ''] + self.assertEqual(0.5, bu.get_lvm_config_value('fake_chroot', + 'section', 'bar')) + + mock_exec.return_value = [r'buzz="spam"', ''] + self.assertEqual("spam", bu.get_lvm_config_value('fake_chroot', + 'section', 'buzz')) + + mock_exec.return_value = [r'list=[1, 2.3, 4., .5, "6", "7", "8"]', ''] + self.assertEqual([1, 2.3, 4., .5, "6", "7", "8"], + bu.get_lvm_config_value('fake_chroot', + 'section', 'list')) + + mock_exec.return_value = [r'ist2=["1", "spam egg", ' + r'"^kind\of\regex?[.$42]"]', ''] + self.assertEqual(["1", "spam egg", r"^kind\of\regex?[.$42]"], + bu.get_lvm_config_value('fake_chroot', + 'section', 'list2')) + + def test_update_raw_config(self): + RAW_CONFIG = ''' +foo { +\tbar=42 +}''' + self.assertEqual(''' +foo { +\tbar=1 +}''', bu._update_option_in_lvm_raw_config('foo', 'bar', 1, RAW_CONFIG)) + self.assertEqual(''' +foo { +\tbar=42 +\tbuzz=1 +}''', bu._update_option_in_lvm_raw_config('foo', 'buzz', 1, RAW_CONFIG)) + self.assertEqual(''' +foo { +\tbar=42 +} +spam { +\tegg=1 +}''', bu._update_option_in_lvm_raw_config('spam', 'egg', 1, RAW_CONFIG)) + self.assertEqual(''' +foo { +\tbar=[1, 2.3, "foo", "buzz"] +}''', bu._update_option_in_lvm_raw_config('foo', 'bar', + [1, 2.3, "foo", "buzz"], + RAW_CONFIG)) + + @mock.patch.object(os, 'remove') + @mock.patch.object(utils, 'execute') + @mock.patch.object(bu, '_update_option_in_lvm_raw_config') + @mock.patch.object(shutil, 'copy') + @mock.patch.object(shutil, 'move') + def test_override_config_value(self, m_move, m_copy, m_upd, m_execute, + m_remove): + m_execute.side_effect = (['old_fake_config', ''], + ['fake_config', '']) + m_upd.return_value = 'fake_config' + with mock.patch('six.moves.builtins.open', create=True) as mock_open: + file_handle_mock = mock_open.return_value.__enter__.return_value + bu.override_lvm_config_value('fake_chroot', + 'foo', 'bar', 'buzz', 'lvm.conf') + file_handle_mock.write.assert_called_with('fake_config') + m_upd.assert_called_once_with('foo', 'bar', 'buzz', 'old_fake_config') + m_copy.assert_called_once_with('lvm.conf', + 'lvm.conf.bak') + + @mock.patch.object(os, 'remove') + @mock.patch.object(utils, 'execute') + @mock.patch.object(bu, '_update_option_in_lvm_raw_config') + @mock.patch.object(shutil, 'copy') + @mock.patch.object(shutil, 'move') + def test_override_config_value_fail(self, m_move, m_copy, m_upd, m_execute, + m_remove): + m_execute.side_effect = (['old_fake_config', ''], + errors.ProcessExecutionError()) + m_upd.return_value = 'fake_config' + with mock.patch('six.moves.builtins.open', create=True) as mock_open: + file_handle_mock = mock_open.return_value.__enter__.return_value + self.assertRaises(errors.ProcessExecutionError, + bu.override_lvm_config_value, + 'fake_chroot', 'foo', 'bar', 'buzz', 'lvm.conf') + self.assertTrue(file_handle_mock.write.called) + m_copy.assert_called_once_with('lvm.conf', + 'lvm.conf.bak') + m_move.assert_called_once_with('lvm.conf.bak', + 'lvm.conf') + + @mock.patch.object(bu, 'get_lvm_config_value') + @mock.patch.object(bu, 'override_lvm_config_value') + def test_append_lvm_devices_filter(self, m_override_config, m_get_config): + m_get_config.return_value = ['fake1'] + bu.append_lvm_devices_filter('fake_chroot', ['fake2', 'fake3']) + m_override_config.assert_called_once_with( + 'fake_chroot', + 'devices', 'filter', + ['fake1', 'fake2', 'fake3'], + 'fake_chroot/etc/lvm/lvm.conf') diff --git a/bareon/tests/test_do_partitioning.py b/bareon/tests/test_do_partitioning.py index 4289883..7a6013a 100644 --- a/bareon/tests/test_do_partitioning.py +++ b/bareon/tests/test_do_partitioning.py @@ -66,6 +66,7 @@ class TestPartitioningAction(unittest2.TestCase): self.assertEqual(mock_fu_mf_expected_calls, mock_fu.make_fs.call_args_list) + @mock.patch.object(partitioning, 'hu', autospec=True) @mock.patch.object(partitioning, 'os', autospec=True) @mock.patch.object(partitioning, 'utils', autospec=True) @mock.patch.object(partitioning, 'mu', autospec=True) @@ -73,7 +74,7 @@ class TestPartitioningAction(unittest2.TestCase): @mock.patch.object(partitioning, 'fu', autospec=True) @mock.patch.object(partitioning, 'pu', autospec=True) def test_do_partitioning_md(self, mock_pu, mock_fu, mock_lu, mock_mu, - mock_utils, mock_os): + mock_utils, mock_os, mock_hu): mock_os.path.exists.return_value = True self.drv.partition_scheme.mds = [ objects.MD('fake_md1', 'mirror', devices=['/dev/sda1', @@ -88,6 +89,7 @@ class TestPartitioningAction(unittest2.TestCase): ['/dev/sdb3', '/dev/sdc1'], 'default')], mock_mu.mdcreate.call_args_list) + @mock.patch.object(partitioning, 'hu', autospec=True) @mock.patch.object(partitioning, 'os', autospec=True) @mock.patch.object(partitioning, 'utils', autospec=True) @mock.patch.object(partitioning, 'mu', autospec=True) @@ -95,7 +97,7 @@ class TestPartitioningAction(unittest2.TestCase): @mock.patch.object(partitioning, 'fu', autospec=True) @mock.patch.object(partitioning, 'pu', autospec=True) def test_do_partitioning(self, mock_pu, mock_fu, mock_lu, mock_mu, - mock_utils, mock_os): + mock_utils, mock_os, mock_hu): mock_os.path.exists.return_value = True self.action.execute() mock_utils.unblacklist_udev_rules.assert_called_once_with( @@ -165,3 +167,91 @@ class TestPartitioningAction(unittest2.TestCase): mock.call('xfs', '', '', '/dev/mapper/image-glance')] self.assertEqual(mock_fu_mf_expected_calls, mock_fu.make_fs.call_args_list) + self.assertEqual([mock.call('/dev/sda'), + mock.call('/dev/sdb'), + mock.call('/dev/sdc')], + mock_hu.is_multipath_device.call_args_list) + + +class TestManagerMultipathPartition(unittest2.TestCase): + + @mock.patch('bareon.drivers.data.nailgun.Nailgun.parse_image_meta', + return_value={}) + @mock.patch('bareon.drivers.data.nailgun.hu.list_block_devices') + def setUp(self, mock_lbd, mock_image_meta): + super(TestManagerMultipathPartition, self).setUp() + mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_MPATH + data = copy.deepcopy(test_nailgun.PROVISION_SAMPLE_DATA) + data['ks_meta']['pm_data']['ks_spaces'] =\ + test_nailgun.MPATH_DISK_KS_SPACES + self.drv = nailgun.Nailgun(data) + self.action = partitioning.PartitioningAction(self.drv) + + @mock.patch.object(partitioning, 'hu', autospec=True) + @mock.patch.object(partitioning, 'os', autospec=True) + @mock.patch.object(partitioning, 'utils', autospec=True) + @mock.patch.object(partitioning, 'mu', autospec=True) + @mock.patch.object(partitioning, 'lu', autospec=True) + @mock.patch.object(partitioning, 'fu', autospec=True) + @mock.patch.object(partitioning, 'pu', autospec=True) + def test_do_partitioning_mp(self, mock_pu, mock_fu, mock_lu, mock_mu, + mock_utils, mock_os, mock_hu): + mock_os.path.exists.return_value = True + mock_hu.list_block_devices.return_value = test_nailgun.\ + LIST_BLOCK_DEVICES_MPATH + self.action._make_partitions = mock.MagicMock() + mock_hu.is_multipath_device.side_effect = [True, False] + seq = mock.Mock() + seq.attach_mock(mock_utils.blacklist_udev_rules, 'blacklist') + seq.attach_mock(mock_utils.unblacklist_udev_rules, 'unblacklist') + seq.attach_mock(self.action._make_partitions, '_make_partitions') + + self.action.execute() + + seq_calls = [ + mock.call.blacklist(udev_rules_dir='/etc/udev/rules.d', + udev_rules_lib_dir='/lib/udev/rules.d', + udev_empty_rule='empty_rule', + udev_rename_substr='.renamedrule'), + mock.call._make_partitions([mock.ANY]), + mock.call.unblacklist(udev_rules_dir='/etc/udev/rules.d', + udev_rename_substr='.renamedrule'), + mock.call._make_partitions([mock.ANY], + wait_for_udev_settle=True)] + self.assertEqual(seq_calls, seq.mock_calls) + + parted_list = seq.mock_calls[1][1][0] + self.assertEqual(parted_list[0].name, '/dev/sdc') + parted_list = seq.mock_calls[3][1][0] + self.assertEqual(parted_list[0].name, '/dev/mapper/12312') + + mock_fu_mf_expected_calls = [ + mock.call('ext2', '', '', '/dev/mapper/12312-part3'), + mock.call('ext4', '', '', '/dev/sdc1')] + self.assertEqual(mock_fu_mf_expected_calls, + mock_fu.make_fs.call_args_list) + + @mock.patch.object(partitioning, 'pu', autospec=True) + @mock.patch.object(partitioning, 'utils', autospec=True) + @mock.patch.object(partitioning, 'os', autospec=True) + def test_make_partitions_settle(self, mock_os, mock_utils, mock_pu): + self.action._make_partitions(self.drv.partition_scheme.parteds, + wait_for_udev_settle=True) + + for call in mock_utils.wait_for_udev_settle.mock_calls: + self.assertEqual(call, mock.call(attempts=10)) + + self.assertEqual(mock_pu.make_label.mock_calls, [ + mock.call('/dev/mapper/12312', 'gpt'), + mock.call('/dev/sdc', 'gpt')]) + + self.assertEqual(mock_pu.make_partition.mock_calls, [ + mock.call('/dev/mapper/12312', 1, 25, 'primary'), + mock.call('/dev/mapper/12312', 25, 225, 'primary'), + mock.call('/dev/mapper/12312', 225, 425, 'primary'), + mock.call('/dev/mapper/12312', 425, 625, 'primary'), + mock.call('/dev/mapper/12312', 625, 645, 'primary'), + mock.call('/dev/sdc', 1, 201, 'primary')]) + + self.assertEqual(mock_pu.set_partition_flag.mock_calls, [ + mock.call('/dev/mapper/12312', 1, 'bios_grub')]) diff --git a/bareon/tests/test_hardware_utils.py b/bareon/tests/test_hardware_utils.py index 792bb89..8b98f2b 100644 --- a/bareon/tests/test_hardware_utils.py +++ b/bareon/tests/test_hardware_utils.py @@ -165,6 +165,60 @@ supports-register-dump: yes '--name=/dev/fake', check_exit_code=[0]) + def test_multipath_true(self): + uspec = { + 'DEVLINKS': ['/dev/disk/by-id/fakeid1', + '/dev/disk/by-id/dm-uuid-mpath-231'], + 'DEVNAME': '/dev/dm-0', + 'DEVPATH': '/devices/fakepath', + 'DEVTYPE': 'disk', + 'MAJOR': '11', + 'MINOR': '0', + 'ID_BUS': 'fakebus', + 'ID_MODEL': 'fakemodel', + 'ID_SERIAL_SHORT': 'fakeserial', + 'ID_WWN': 'fakewwn', + 'ID_CDROM': '1' + } + self.assertEqual(True, hu.is_multipath_device('/dev/mapper/231', + uspec)) + + def test_multipath_false(self): + uspec = { + 'DEVLINKS': ['/dev/disk/by-id/fakeid1', + '/dev/disk/by-id/dm-name-fakeid1'], + 'DEVNAME': '/dev/dm-0', + 'DEVPATH': '/devices/fakepath', + 'DEVTYPE': 'disk', + 'MAJOR': '11', + 'MINOR': '0', + 'ID_BUS': 'fakebus', + 'ID_MODEL': 'fakemodel', + 'ID_SERIAL_SHORT': 'fakeserial', + 'ID_WWN': 'fakewwn', + 'ID_CDROM': '1' + } + self.assertEqual(False, hu.is_multipath_device('/dev/sda', uspec)) + + @mock.patch.object(hu, 'udevreport') + def test_multipath_no_uspec(self, mock_udev): + uspec = { + 'DEVLINKS': ['/dev/disk/by-id/fakeid1', + '/dev/disk/by-id/dm-uuid-mpath-231'], + 'DEVNAME': '/dev/dm-0', + 'DEVPATH': '/devices/fakepath', + 'DEVTYPE': 'disk', + 'MAJOR': '11', + 'MINOR': '0', + 'ID_BUS': 'fakebus', + 'ID_MODEL': 'fakemodel', + 'ID_SERIAL_SHORT': 'fakeserial', + 'ID_WWN': 'fakewwn', + 'ID_CDROM': '1' + } + mock_udev.return_value = uspec + self.assertEqual(True, hu.is_multipath_device('/dev/mapper/231')) + @mock.patch.object(utils, 'execute') def test_blockdevreport(self, mock_exec): # should run blockdev OS command diff --git a/bareon/tests/test_nailgun.py b/bareon/tests/test_nailgun.py index dcb3bfd..4c1cada 100644 --- a/bareon/tests/test_nailgun.py +++ b/bareon/tests/test_nailgun.py @@ -515,6 +515,100 @@ LIST_BLOCK_DEVICES_SAMPLE_NVME = [ 'size': 500107862016}, ] +LIST_BLOCK_DEVICES_MPATH = [ + {'uspec': + {'DEVLINKS': [ + 'disk/by-id/scsi-SATA_VBOX_HARDDISK_VB69050467-b385c7cd', + '/dev/disk/by-id/wwn-fake_wwn_1', + '/dev/disk/by-path/pci-0000:00:1f.2-scsi-0:0:0:0'], + 'ID_SERIAL_SHORT': 'fake_serial_1', + 'ID_WWN': 'fake_wwn_1', + 'DEVPATH': '/devices/pci0000:00/0000:00:1f.2/ata1/host0/' + 'target0:0:0/0:0:0:0/block/sda', + 'ID_MODEL': 'fake_id_model', + 'DEVNAME': '/dev/sda', + 'MAJOR': '8', + 'DEVTYPE': 'disk', 'MINOR': '0', 'ID_BUS': 'ata' + }, + 'startsec': '0', + 'device': '/dev/sda', + 'espec': {'state': 'running', 'timeout': '30', 'removable': '0'}, + 'bspec': { + 'sz': '976773168', 'iomin': '4096', 'size64': '500107862016', + 'ss': '512', 'ioopt': '0', 'alignoff': '0', 'pbsz': '4096', + 'ra': '256', 'ro': '0', 'maxsect': '1024' + }, + 'size': 500107862016}, + {'uspec': + {'DEVLINKS': [ + 'disk/by-id/scsi-SATA_VBOX_HARDDISK_VB69050467-b385c7cd', + '/dev/disk/by-id/wwn-fake_wwn_1', + '/dev/disk/by-path/pci-0000:00:1f.2-scsi-0:0:1:0'], + 'ID_SERIAL_SHORT': 'fake_serial_1', + 'ID_WWN': 'fake_wwn_1', + 'DEVPATH': '/devices/pci0000:00/0000:00:1f.2/ata1/host0/' + 'target0:0:0/0:0:0:0/block/sdb', + 'ID_MODEL': 'fake_id_model', + 'DEVNAME': '/dev/sdb', + 'MAJOR': '8', + 'DEVTYPE': 'disk', 'MINOR': '0', 'ID_BUS': 'ata' + }, + 'startsec': '0', + 'device': '/dev/sdb', + 'espec': {'state': 'running', 'timeout': '30', 'removable': '0'}, + 'bspec': { + 'sz': '976773168', 'iomin': '4096', 'size64': '500107862016', + 'ss': '512', 'ioopt': '0', 'alignoff': '0', 'pbsz': '4096', + 'ra': '256', 'ro': '0', 'maxsect': '1024' + }, + 'size': 500107862016}, + {'uspec': + {'DEVLINKS': [ + 'disk/by-id/scsi-SATA_VBOX_HARDDISK_VB69050467-b385c7cd', + '/dev/disk/by-id/wwn-fake_wwn_1', + '/dev/disk/by-id/dm-uuid-mpath-fake_wwn_1' + ], + 'ID_SERIAL_SHORT': 'fake_serial_1', + 'ID_WWN': 'fake_wwn_1', + 'DEVPATH': '/devices/pci0000:00/0000:00:1f.2/ata1/host0/', + 'ID_MODEL': 'fake_id_model', + 'DEVNAME': '/dev/dm-0', + 'MAJOR': '8', + 'DEVTYPE': 'disk', 'MINOR': '0', 'ID_BUS': 'ata' + }, + 'startsec': '0', + 'device': '/dev/mapper/12312', + 'espec': {'state': 'running', 'timeout': '30', 'removable': '0'}, + 'bspec': { + 'sz': '976773168', 'iomin': '4096', 'size64': '500107862016', + 'ss': '512', 'ioopt': '0', 'alignoff': '0', 'pbsz': '4096', + 'ra': '256', 'ro': '0', 'maxsect': '1024' + }, + 'size': 500107862016}, + {'uspec': + {'DEVLINKS': [ + 'disk/by-id/scsi-SATA_VBOX_HARDDISK_VB69050467-fffff', + '/dev/disk/by-id/wwn-fake_wwn_2' + '/dev/disk/by-path/pci-0000:00:1f.2-scsi-0:0:4:0'], + 'ID_SERIAL_SHORT': 'fake_serial_2', + 'ID_WWN': 'fake_wwn_1', + 'DEVPATH': '/devices/pci0000:00/0000:00:1f.2/ata2/host1/', + 'ID_MODEL': 'fake_id_model', + 'DEVNAME': '/dev/sdc', + 'MAJOR': '8', + 'DEVTYPE': 'disk', 'MINOR': '0', 'ID_BUS': 'ata' + }, + 'startsec': '0', + 'device': '/dev/sdc', + 'espec': {'state': 'running', 'timeout': '30', 'removable': '0'}, + 'bspec': { + 'sz': '976773168', 'iomin': '4096', 'size64': '500107862016', + 'ss': '512', 'ioopt': '0', 'alignoff': '0', 'pbsz': '4096', + 'ra': '256', 'ro': '0', 'maxsect': '1024' + }, + 'size': 500107862016}, +] + SINGLE_DISK_KS_SPACES = [ { "name": "sda", @@ -593,6 +687,58 @@ SECOND_DISK_OS_KS_SPACES = [ } ] +MPATH_DISK_KS_SPACES = [ + { + "name": "mapper/12312", + "extra": [ + 'disk/by-id/scsi-SATA_VBOX_HARDDISK_VB69050467-b385c7cd', + 'disk/by-id/wwn-fake_wwn_1'], + "free_space": 1024, + "volumes": [ + { + "type": "boot", + "size": 300 + }, + { + "mount": "/boot", + "size": 200, + "type": "partition", + "file_system": "ext2", + "name": "Boot" + }, + { + "mount": "/", + "size": 200, + "type": "partition", + "file_system": "ext4", + "name": "Root", + }, + ], + "type": "disk", + "id": "dm-0", + "size": 102400 + }, + { + "name": "sdc", + "extra": [ + 'disk/by-id/scsi-SATA_VBOX_HARDDISK_VB69050467-fffff', + 'disk/by-id/wwn-fake_wwn_2'], + "free_space": 1024, + "volumes": [ + { + "mount": "/home", + "size": 200, + "type": "partition", + "file_system": "ext4", + "name": "Root", + }, + ], + "type": "disk", + "id": "sdc", + "size": 102400 + } +] + NO_BOOT_KS_SPACES = [ { "name": "sda", diff --git a/bareon/tests/test_partition.py b/bareon/tests/test_partition.py index da0d3aa..f996dee 100644 --- a/bareon/tests/test_partition.py +++ b/bareon/tests/test_partition.py @@ -271,6 +271,16 @@ class TestParted(unittest2.TestCase): expected_name = '%sp%s' % (self.prtd.name, 1) self.assertEqual(expected_name, self.prtd.next_name()) + @mock.patch.object(objects.Parted, 'next_count') + @mock.patch.object(objects.Parted, 'next_type') + def test_next_name_with_separator_part(self, nt_mock, nc_mock): + nc_mock.return_value = 2 + nt_mock.return_value = 'not_extended' + self.prtd.name = '/dev/mapper/123' + + expected_name = '/dev/mapper/123-part2' + self.assertEqual(expected_name, self.prtd.next_name()) + def test_next_begin_empty_partitions(self): self.assertEqual(1, self.prtd.next_begin()) diff --git a/bareon/utils/build.py b/bareon/utils/build.py index 3ac015a..0b4086e 100644 --- a/bareon/utils/build.py +++ b/bareon/utils/build.py @@ -876,3 +876,134 @@ def save_bs_container(output, input_dir, format="tar.gz"): raise errors.WrongOutputContainer( "Unsupported bootstrap container format {0}." .format(format)) + + +# NOTE(sslypushenko) Modern lvm supports lvmlocal.conf to selective overriding +# set of configuration options. So, this functionality for patching lvm +# configuration should be removed after lvm upgrade in Ubuntu repositories and +# replaced with proper lvmlocal.conf file +def get_lvm_config_value(chroot, section, name): + """Get option value from current lvm configuration. + + If option is not present in lvm.conf, None returns + """ + raw_value = utils.execute('chroot', chroot, 'lvm dumpconfig', + '/'.join((section, name)), + check_exit_code=[0, 5])[0] + if '=' not in raw_value: + return + + raw_value = raw_value.split('=')[1].strip() + + re_str = '"[^"]*"' + re_float = '\\d*\\.\\d*' + re_int = '\\d+' + tokens = re.findall('|'.join((re_str, re_float, re_int)), raw_value) + + values = [] + for token in tokens: + if re.match(re_str, token): + values.append(token.strip('"')) + elif re.match(re_float, token): + values.append(float(token)) + elif re.match(re_int, token): + values.append(int(token)) + + if not values: + return + elif len(values) == 1: + return values[0] + else: + return values + + +def _update_option_in_lvm_raw_config(section, name, value, raw_config): + """Update option in dumped LVM configuration. + + :param raw_config should be a string with dumped LVM configuration. + + If section and key present in config, option will be overwritten. + If there is no key but section presents in config, option will be added + in to the end of section. + If there are no section and key in config, section will be added in the end + of the config. + """ + def dump_value(value): + if isinstance(value, int): + return str(value) + elif isinstance(value, float): + return '{:.10f}'.format(value).rstrip('0') + elif isinstance(value, str): + return '"{}"'.format(value) + elif isinstance(value, list or tuple): + return '[{}]'.format(', '.join(dump_value(v) for v in value)) + + lines = raw_config.splitlines() + section_start = next((n for n, line in enumerate(lines) + if line.strip().startswith('{} '.format(section))), + None) + if section_start is None: + raw_section = '{} {{\n\t{}={}\n}}'.format(section, name, + dump_value(value)) + lines.append(raw_section) + return '\n'.join(lines) + + line_no = section_start + while not lines[line_no].strip().endswith('}'): + if lines[line_no].strip().startswith(name): + lines[line_no] = '\t{}={}'.format(name, dump_value(value)) + return '\n'.join(lines) + line_no += 1 + + lines[line_no] = '\t{}={}\n}}'.format(name, dump_value(value)) + return '\n'.join(lines) + + +def override_lvm_config_value(chroot, section, name, value, lvm_conf_file): + """Override option in LVM configuration. + + If option is not valid, then errors.ProcessExecutionError will be raised + and lvm configuration will remain unchanged + """ + updated_config = _update_option_in_lvm_raw_config( + section, name, value, + utils.execute('chroot', chroot, 'lvm dumpconfig')[0]) + lvm_conf_file_bak = '.'.join((lvm_conf_file, 'bak')) + shutil.copy(lvm_conf_file, lvm_conf_file_bak) + LOG.debug('Backup for origin LVM configuration file: {}' + ''.format(lvm_conf_file_bak)) + with open(lvm_conf_file, mode='w') as lvm_conf: + lvm_conf.write(updated_config) + + # NOTE(sslypushenko) Extra cycle of dump/save lvm.conf is required to be + # sure that updated configuration is valid and to adjust it to general + # lvm.conf formatting + try: + current_config = utils.execute('chroot', chroot, 'lvm dumpconfig')[0] + with open(lvm_conf_file, mode='w') as lvm_conf: + lvm_conf.write(current_config) + LOG.info('LVM configuration updated') + except errors.ProcessExecutionError as exc: + shutil.move(lvm_conf_file_bak, lvm_conf_file) + LOG.debug('Option {}/{} can not be updated with value {}.' + ''.format(section, name, value)) + raise exc + + +def append_lvm_devices_filter(chroot, multipath_lvm_filter, + lvm_conf_path='/etc/lvm/lvm.conf'): + """Append custom devises filters to LVM configuration + + LVM configuration should de updated to force lvm work with partitions + on multipath devices using /dev/mapper/-part links. + Other links to these devices should be blacklisted in devices/filter + option + """ + lvm_filter = get_lvm_config_value(chroot, 'devices', 'filter') or [] + if isinstance(lvm_filter, str): + lvm_filter = [lvm_filter] + lvm_filter = set(lvm_filter) + lvm_filter.update(multipath_lvm_filter) + override_lvm_config_value(chroot, 'devices', 'filter', + sorted(list(lvm_filter)), + os.path.join(chroot, lvm_conf_path.lstrip('/'))) diff --git a/bareon/utils/hardware.py b/bareon/utils/hardware.py index 4908f77..cf1f4e6 100644 --- a/bareon/utils/hardware.py +++ b/bareon/utils/hardware.py @@ -276,6 +276,14 @@ def scsi_address(dev): return address +def is_multipath_device(device, uspec=None): + """Check whether block device with given uspec is multipath device""" + if uspec is None: + uspec = udevreport(device) + return any((devlink.startswith('/dev/disk/by-id/dm-uuid-mpath-') + for devlink in uspec.get('DEVLINKS', []))) + + def get_block_devices_from_udev_db(): return get_block_data_from_udev('disk') @@ -400,6 +408,12 @@ def get_device_info(device, disks=True): if disks and not is_disk(device, bspec=bspec, uspec=uspec): return + # NOTE(kszukielojc) if block device is multipath device, + # devlink /dev/mapper/* should be used instead /dev/dm-* + if is_multipath_device(device, uspec=uspec): + device = [devlink for devlink in uspec['DEVLINKS'] + if devlink.startswith('/dev/mapper/')][0] + bdev = { 'device': device, # NOTE(agordeev): blockdev gets 'startsec' from sysfs, diff --git a/bareon/utils/utils.py b/bareon/utils/utils.py index d80f868..3c7203c 100644 --- a/bareon/utils/utils.py +++ b/bareon/utils/utils.py @@ -319,6 +319,7 @@ def blacklist_udev_rules(udev_rules_dir, udev_rules_lib_dir, so we should increase processing speed for those events, otherwise partitioning is doomed. """ + LOG.debug("Enabling udev's rules blacklisting") empty_rule_path = os.path.join(udev_rules_dir, os.path.basename(udev_empty_rule)) with open(empty_rule_path, 'w') as f: @@ -345,6 +346,7 @@ def blacklist_udev_rules(udev_rules_dir, udev_rules_lib_dir, def unblacklist_udev_rules(udev_rules_dir, udev_rename_substr): """disable udev's rules blacklisting""" + LOG.debug("Disabling udev's rules blacklisting") for rule in os.listdir(udev_rules_dir): src = os.path.join(udev_rules_dir, rule) if os.path.isdir(src): @@ -373,6 +375,17 @@ def unblacklist_udev_rules(udev_rules_dir, udev_rename_substr): udevadm_settle() +def wait_for_udev_settle(attempts): + """Wait for emptiness of udev queue within attempts*0.1 seconds""" + for attempt in six.moves.range(attempts): + try: + udevadm_settle() + except errors.ProcessExecutionError: + LOG.warning("udevadm settle did return non-zero exit code. " + "Partitioning continues.") + time.sleep(0.1) + + def parse_kernel_cmdline(): """Parse linux kernel command line""" with open('/proc/cmdline', 'rt') as f: