From 753a87c4bb7ffb6baaaa446fe4b2e7750f40483f Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Tue, 14 Jul 2015 14:25:08 +0300 Subject: [PATCH] fuel-agent: use different md metadata version 0.90 will be used for centos only letting GRUB-legacy to boot. Otherwise, default (1.2) will be used as GRUB2 is able to deal with it. Change-Id: I89c5178d991970de4682c1e239e0ecb014ba61bd Closes-Bug: #1430432 --- fuel_agent/drivers/nailgun.py | 84 +++++- fuel_agent/manager.py | 11 +- fuel_agent/objects/__init__.py | 2 + fuel_agent/objects/operating_system.py | 13 +- fuel_agent/objects/partition/md.py | 5 +- fuel_agent/objects/partition/scheme.py | 1 + fuel_agent/tests/test_manager.py | 44 +++- fuel_agent/tests/test_md_utils.py | 2 +- fuel_agent/tests/test_nailgun.py | 245 +++++++++++------- fuel_agent/tests/test_nailgun_build_image.py | 9 +- fuel_agent/tests/test_partition.py | 1 + .../tests/test_simple_nailgun_driver.py | 2 + fuel_agent/utils/md.py | 4 +- 13 files changed, 302 insertions(+), 121 deletions(-) diff --git a/fuel_agent/drivers/nailgun.py b/fuel_agent/drivers/nailgun.py index a8e8e61..fa0bc2a 100644 --- a/fuel_agent/drivers/nailgun.py +++ b/fuel_agent/drivers/nailgun.py @@ -80,9 +80,13 @@ class Nailgun(BaseDataDriver): # has already been added. we need this to # get rid of md over all disks for /boot partition. self._boot_done = False + self._image_meta = self.parse_image_meta() - self._partition_scheme = self.parse_partition_scheme() + self._operating_system = self.parse_operating_system() self._grub = self.parse_grub() + # parsing partition scheme needs grub and operating system have + # been parsed + self._partition_scheme = self.parse_partition_scheme() self._configdrive_scheme = self.parse_configdrive_scheme() # parsing image scheme needs partition scheme has been parsed self._image_scheme = self.parse_image_scheme() @@ -99,9 +103,14 @@ class Nailgun(BaseDataDriver): def grub(self): return self._grub + @property + def have_grub1_by_default(self): + return (isinstance(self.operating_system, objects.Centos) and + self.operating_system.major == 6) + @property def operating_system(self): - return None + return self._operating_system @property def configdrive_scheme(self): @@ -174,6 +183,49 @@ class Nailgun(BaseDataDriver): def _num_ceph_osds(self): return self._get_partition_count('ceph') + def get_os_by_image_meta(self, os_release): + LOG.debug('--- Getting operating system data by image metadata ---') + if os_release: + LOG.debug('Looks like %{0} is going to be provisioned'. + format(os_release)) + try: + OS = getattr(objects, os_release['name']) + os = OS(repos=None, packages=None, major=os_release['major'], + minor=os_release['minor']) + return os + except (AttributeError, KeyError): + LOG.warning('Cannot guess operating system release ' + 'from image metadata') + + def get_os_by_profile(self, profile): + LOG.debug('--- Getting operating system data by profile ---') + if 'centos' in profile: + os = objects.Centos(repos=None, packages=None, major=6, minor=5) + if '7' in profile: + LOG.debug('Looks like CentOS7.0 is going to be provisioned.') + os = objects.Centos(repos=None, packages=None, major=7, + minor=0) + else: + LOG.debug('Looks like CentOS6.5 is going to be provisioned.') + return os + elif 'ubuntu' in profile: + os = objects.Ubuntu(repos=None, packages=None, major=12, minor=4) + if '1404' in profile: + LOG.debug('Looks like Ubuntu1404 is going to be provisioned.') + os = objects.Ubuntu(repos=None, packages=None, major=14, + minor=4) + else: + LOG.debug('Looks like Ubuntu1204 is going to be provisioned.') + return os + os = objects.OperatingSystem(repos=None, packages=None) + return os + + def parse_operating_system(self): + LOG.debug('--- Preparing operating system data ---') + os_release = self._image_meta.get('os', None) + return self.get_os_by_image_meta(os_release) or \ + self.get_os_by_profile(self.data['profile'].lower()) + def parse_partition_scheme(self): LOG.debug('--- Preparing partition scheme ---') data = self.partition_data() @@ -344,10 +396,18 @@ class Nailgun(BaseDataDriver): volume['mount'] not in ('none', '/boot'): LOG.debug('Attaching partition to RAID ' 'by its mount point %s' % volume['mount']) + metadata = 'default' + if self.have_grub1_by_default: + metadata = '0.90' + LOG.debug('Going to use MD metadata version {0}. ' + 'The version was guessed at the data has ' + 'been given about the operating system.' + .format(metadata)) partition_scheme.md_attach_by_mount( device=prt.name, mount=volume['mount'], fs_type=volume.get('file_system', 'xfs'), - fs_label=self._getlabel(volume.get('disk_label'))) + fs_label=self._getlabel(volume.get('disk_label')), + metadata=metadata) if 'mount' in volume and volume['mount'] == '/boot' and \ not self._boot_done: @@ -469,12 +529,13 @@ class Nailgun(BaseDataDriver): LOG.debug('Prefered kernel version is 2.6') grub.kernel_regexp = r'^vmlinuz-2\.6.*' grub.initrd_regexp = r'^initramfs-2\.6.*' + grub.version = 1 if self.have_grub1_by_default else 2 + LOG.debug('Grub version is %{0}'.format(grub.version)) return grub - def parse_image_scheme(self): - LOG.debug('--- Preparing image scheme ---') + def parse_image_meta(self): + LOG.debug('--- Preparing image metadata ---') data = self.data - image_scheme = objects.ImageScheme() # FIXME(agordeev): this piece of code for fetching additional image # meta data should be factored out of this particular nailgun driver # into more common and absract data getter which should be able to deal @@ -496,6 +557,13 @@ class Nailgun(BaseDataDriver): LOG.exception(e) LOG.debug('Failed to fetch/decode image meta data') image_meta = {} + return image_meta + + def parse_image_scheme(self): + LOG.debug('--- Preparing image scheme ---') + data = self.data + image_meta = self._image_meta + image_scheme = objects.ImageScheme() # We assume for every file system user may provide a separate # file system image. For example if partitioning scheme has # /, /boot, /var/lib file systems then we will try to get images @@ -619,8 +687,8 @@ class NailgunBuildImage(BaseDataDriver): suite=repo['suite'], section=repo['section'], priority=repo['priority'])) - - return objects.Ubuntu(repos=repos, packages=packages) + os = objects.Ubuntu(repos=repos, packages=packages, major=14, minor=4) + return os def parse_schemes(self): diff --git a/fuel_agent/manager.py b/fuel_agent/manager.py index f5160cb..13a495f 100644 --- a/fuel_agent/manager.py +++ b/fuel_agent/manager.py @@ -233,7 +233,7 @@ class Manager(object): # creating meta disks for md in self.driver.partition_scheme.mds: - mu.mdcreate(md.name, md.level, md.devices) + mu.mdcreate(md.name, md.level, md.devices, md.metadata) # creating physical volumes for pv in self.driver.partition_scheme.pvs: @@ -414,7 +414,12 @@ class Manager(object): grub = self.driver.grub - grub.version = gu.guess_grub_version(chroot=chroot) + guessed_version = gu.guess_grub_version(chroot=chroot) + if guessed_version != grub.version: + grub.version = guessed_version + LOG.warning('Grub version differs from which the operating system ' + 'should have by default. Found version in image: ' + '{0}'.format(guessed_version)) boot_device = self.driver.partition_scheme.boot_device(grub.version) install_devices = [d.name for d in self.driver.partition_scheme.parteds if d.install_bootloader] @@ -523,6 +528,8 @@ class Manager(object): # as a pluggable data driver to avoid any fixed format. metadata = {} + metadata['os'] = self.driver.operating_system.to_dict() + # TODO(kozhukalov): implement this using image metadata # we need to compare list of packages and repos LOG.info('*** Checking if image exists ***') diff --git a/fuel_agent/objects/__init__.py b/fuel_agent/objects/__init__.py index 7c23127..561aa4e 100644 --- a/fuel_agent/objects/__init__.py +++ b/fuel_agent/objects/__init__.py @@ -19,6 +19,7 @@ from fuel_agent.objects.configdrive import ConfigDriveScheme from fuel_agent.objects.device import Loop from fuel_agent.objects.image import Image from fuel_agent.objects.image import ImageScheme +from fuel_agent.objects.operating_system import Centos from fuel_agent.objects.operating_system import OperatingSystem from fuel_agent.objects.operating_system import Ubuntu from fuel_agent.objects.partition.fs import FileSystem @@ -63,6 +64,7 @@ __all__ = [ 'Grub', 'OperatingSystem', 'Ubuntu', + 'Centos', 'Repo', 'DEBRepo', 'Loop', diff --git a/fuel_agent/objects/operating_system.py b/fuel_agent/objects/operating_system.py index 3eac4ab..e7eda50 100644 --- a/fuel_agent/objects/operating_system.py +++ b/fuel_agent/objects/operating_system.py @@ -14,10 +14,21 @@ class OperatingSystem(object): - def __init__(self, repos, packages): + def __init__(self, repos, packages, major='unknown', minor='unknown'): self.repos = repos self.packages = packages + self.major = major + self.minor = minor + + def to_dict(self): + return {'major': self.major, + 'minor': self.minor, + 'name': self.__class__.__name__} class Ubuntu(OperatingSystem): pass + + +class Centos(OperatingSystem): + pass diff --git a/fuel_agent/objects/partition/md.py b/fuel_agent/objects/partition/md.py index 3a12175..4b310ee 100644 --- a/fuel_agent/objects/partition/md.py +++ b/fuel_agent/objects/partition/md.py @@ -20,12 +20,14 @@ from fuel_agent.objects import base class MultipleDevice(base.Serializable): def __init__(self, name, level, - devices=None, spares=None, keep_data=False): + devices=None, spares=None, keep_data=False, + metadata='default'): self.keep_data = keep_data self.name = name self.level = level self.devices = devices or [] self.spares = spares or [] + self.metadata = metadata def add_device(self, device): if device in self.devices or device in self.spares: @@ -48,4 +50,5 @@ class MultipleDevice(base.Serializable): 'devices': self.devices, 'spares': self.spares, 'keep_data': self.keep_data, + 'metadata': self.metadata, } diff --git a/fuel_agent/objects/partition/scheme.py b/fuel_agent/objects/partition/scheme.py index edb8fdc..6a765c7 100644 --- a/fuel_agent/objects/partition/scheme.py +++ b/fuel_agent/objects/partition/scheme.py @@ -67,6 +67,7 @@ class PartitionScheme(object): mdkwargs = {} mdkwargs['name'] = kwargs.get('name') or self.md_next_name() mdkwargs['level'] = kwargs.get('level') or 'mirror' + mdkwargs['metadata'] = kwargs.get('metadata') or 'default' md = MultipleDevice(**mdkwargs) self.mds.append(md) return md diff --git a/fuel_agent/tests/test_manager.py b/fuel_agent/tests/test_manager.py index b946d9e..7b855b2 100644 --- a/fuel_agent/tests/test_manager.py +++ b/fuel_agent/tests/test_manager.py @@ -38,10 +38,10 @@ CONF = cfg.CONF class TestManager(unittest2.TestCase): - @mock.patch('yaml.load') - @mock.patch.object(utils, 'init_http_request') + @mock.patch('fuel_agent.drivers.nailgun.Nailgun.parse_image_meta', + return_value={}) @mock.patch.object(hu, 'list_block_devices') - def setUp(self, mock_lbd, mock_http, mock_yaml): + def setUp(self, mock_lbd, mock_image_meta): super(TestManager, self).setUp() mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE self.mgr = manager.Manager(test_nailgun.PROVISION_SAMPLE_DATA) @@ -123,6 +123,7 @@ class TestManager(unittest2.TestCase): else: return ('FAKE_UUID', None) mock_utils.execute.side_effect = _fake_uuid + mock_grub.version = 2 mock_gu.guess_grub_version.return_value = 2 mock_grub.kernel_name = 'fake_kernel_name' mock_grub.initrd_name = 'fake_initrd_name' @@ -145,6 +146,22 @@ class TestManager(unittest2.TestCase): self.assertRaises(errors.WrongPartitionSchemeError, self.mgr.do_bootloader) + @mock.patch('fuel_agent.manager.open', + create=True, new_callable=mock.mock_open) + @mock.patch('fuel_agent.manager.gu', create=True) + @mock.patch('fuel_agent.manager.utils', create=True) + @mock.patch.object(manager.Manager, 'mount_target') + @mock.patch.object(manager.Manager, 'umount_target') + def test_do_bootloader_grub_version_changes( + self, mock_umount, mock_mount, mock_utils, mock_gu, mock_open): + # actually covers only grub1 related logic + mock_utils.execute.return_value = ('fake_UUID\n', None) + mock_gu.guess_grub_version.return_value = 'expected_version' + self.mgr.do_bootloader() + mock_gu.guess_grub_version.assert_called_once_with( + chroot='/tmp/target') + self.assertEqual('expected_version', self.mgr.driver.grub.version) + @mock.patch('fuel_agent.manager.open', create=True, new_callable=mock.mock_open) @mock.patch('fuel_agent.manager.gu', create=True) @@ -245,12 +262,12 @@ class TestManager(unittest2.TestCase): mock_utils.makedirs_if_not_exists.assert_called_once_with( '/tmp/target/etc/nailgun-agent') - @mock.patch('yaml.load') - @mock.patch.object(utils, 'init_http_request') + @mock.patch('fuel_agent.drivers.nailgun.Nailgun.parse_image_meta', + return_value={}) @mock.patch.object(hu, 'list_block_devices') @mock.patch.object(fu, 'make_fs') def test_do_partitioning_with_keep_data_flag(self, mock_fu_mf, mock_lbd, - mock_http, mock_yaml): + mock_image_meta): mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE data = copy.deepcopy(test_nailgun.PROVISION_SAMPLE_DATA) @@ -305,9 +322,9 @@ class TestManager(unittest2.TestCase): ] self.mgr.do_partitioning() self.assertEqual([mock.call('fake_md1', 'mirror', - ['/dev/sda1', '/dev/sdb1']), + ['/dev/sda1', '/dev/sdb1'], 'default'), mock.call('fake_md2', 'mirror', - ['/dev/sdb3', '/dev/sdc1'])], + ['/dev/sdb3', '/dev/sdc1'], 'default')], mock_mu_m.call_args_list) @mock.patch('six.moves.builtins.open') @@ -395,6 +412,9 @@ class TestManager(unittest2.TestCase): mock.call('xfs', '', '', '/dev/mapper/image-glance')] self.assertEqual(mock_fu_mf_expected_calls, mock_fu_mf.call_args_list) + @mock.patch('fuel_agent.drivers.nailgun.Nailgun.parse_image_meta', + return_value={}) + @mock.patch('fuel_agent.drivers.nailgun.Nailgun.parse_operating_system') @mock.patch.object(utils, 'calculate_md5') @mock.patch('os.path.getsize') @mock.patch('yaml.load') @@ -403,7 +423,8 @@ class TestManager(unittest2.TestCase): @mock.patch.object(utils, 'render_and_save') @mock.patch.object(hu, 'list_block_devices') def test_do_configdrive(self, mock_lbd, mock_u_ras, mock_u_e, - mock_http_req, mock_yaml, mock_get_size, mock_md5): + mock_http_req, mock_yaml, mock_get_size, mock_md5, + mock_parse_os, mock_image_meta): mock_get_size.return_value = 123 mock_md5.return_value = 'fakemd5' mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE @@ -755,7 +776,8 @@ class TestImageBuild(unittest2.TestCase): objects.DEBRepo('mos', 'http://fakemos', 'mosX.Y', 'fakesection', priority=1000)], packages=['fakepackage1', 'fakepackage2']) - + self.mgr.driver.operating_system.minor = 4 + self.mgr.driver.operating_system.major = 14 mock_os.path.exists.return_value = False mock_os.path.join.return_value = '/tmp/imgdir/proc' mock_os.path.basename.side_effect = ['img.img.gz', 'img-boot.img.gz'] @@ -881,7 +903,7 @@ class TestImageBuild(unittest2.TestCase): mock.call('/tmp/img-boot.gz', '/fake/img-boot.img.gz')], mock_shutil_move.call_args_list) - metadata = {} + metadata = {'os': {'name': 'Ubuntu', 'major': 14, 'minor': 4}} for repo in self.mgr.driver.operating_system.repos: metadata.setdefault('repos', []).append({ 'type': 'deb', diff --git a/fuel_agent/tests/test_md_utils.py b/fuel_agent/tests/test_md_utils.py index f9874a0..f5437b0 100644 --- a/fuel_agent/tests/test_md_utils.py +++ b/fuel_agent/tests/test_md_utils.py @@ -159,7 +159,7 @@ localhost.localdomain) self.assertEqual(mock_mdclean_expected_calls, mock_mdclean.call_args_list) mock_exec.assert_called_once_with( - 'mdadm', '--create', '--force', '/dev/md0', '-e0.90', + 'mdadm', '--create', '--force', '/dev/md0', '-e', 'default', '--level=mirror', '--raid-devices=2', '/dev/fake1', '/dev/fake2', check_exit_code=[0]) diff --git a/fuel_agent/tests/test_nailgun.py b/fuel_agent/tests/test_nailgun.py index b00c5e7..ebe772f 100644 --- a/fuel_agent/tests/test_nailgun.py +++ b/fuel_agent/tests/test_nailgun.py @@ -20,8 +20,8 @@ import yaml from fuel_agent.drivers import nailgun from fuel_agent import errors +from fuel_agent import objects from fuel_agent.objects import image -from fuel_agent.utils import hardware as hu from fuel_agent.utils import utils @@ -502,6 +502,37 @@ NO_BOOT_KS_SPACES = [ } ] +MD_RAID_KS_SPACES = [ + { + "name": "sda", + "extra": ["sda"], + "free_space": 1024, + "volumes": [ + { + "type": "boot", + "size": 300 + }, + { + "mount": "/boot", + "size": 200, + "type": "raid", + "file_system": "ext2", + "name": "Boot" + }, + { + "mount": "/", + "size": 200, + "type": "raid", + "file_system": "ext4", + "name": "Root" + }, + ], + "type": "disk", + "id": "sda", + "size": 102400 + } +] + FIRST_DISK_HUGE_KS_SPACES = [ { "name": "sda", @@ -711,8 +742,7 @@ MANY_HUGE_DISKS_KS_SPACES = [ ] -class TestNailgun(unittest2.TestCase): - +class TestNailgunMatch(unittest2.TestCase): def test_match_device_by_id_matches(self): # matches by 'by-id' links fake_ks_disk = { @@ -824,10 +854,46 @@ class TestNailgun(unittest2.TestCase): } self.assertFalse(nailgun.match_device(fake_hu_disk, fake_ks_disk)) - @mock.patch('yaml.load') - @mock.patch.object(utils, 'init_http_request') - @mock.patch.object(hu, 'list_block_devices') - def test_configdrive_scheme(self, mock_lbd, mock_http, mock_yaml): + +@mock.patch.object(nailgun.Nailgun, '__init__', return_value=None) +class TestNailgunGetOSMethods(unittest2.TestCase): + def test_parse_operating_system_test_profiles(self, mock_nailgun): + d = {'centos-x86_64': {'obj': objects.Centos, 'minor': 5, 'major': 6}, + 'centos7-x86_64': {'obj': objects.Centos, 'minor': 0, 'major': 7}, + 'ubuntu_1204_x86_64': {'obj': objects.Ubuntu, + 'minor': 4, 'major': 12}, + 'ubuntu_1404_x86_64': {'obj': objects.Ubuntu, + 'minor': 4, 'major': 14}, + 'generic_os': {'obj': objects.OperatingSystem, + 'minor': 'unknown', 'major': 'unknown'}} + drv = nailgun.Nailgun('fake_data') + for profile, obj in d.iteritems(): + os = drv.get_os_by_profile(profile) + self.assertIsInstance(os, obj['obj']) + self.assertEqual(obj['minor'], os.minor) + self.assertEqual(obj['major'], os.major) + + def test_parse_operating_system_image_meta(self, mock_nailgun): + d = {'Centos': objects.Centos, + 'Ubuntu': objects.Ubuntu, + 'unknown': None} + drv = nailgun.Nailgun('fake_data') + for os_name, obj in d.iteritems(): + os = drv.get_os_by_image_meta( + {'name': os_name, 'minor': 1, 'major': 2}) + if os: + self.assertIsInstance(os, obj) + self.assertEqual(1, os.minor) + self.assertEqual(2, os.major) + else: + self.assertIsNone(os) + self.assertEqual('unknown', os_name) + + +@mock.patch.object(nailgun.Nailgun, 'parse_image_meta', return_value={}) +@mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices') +class TestNailgunMockedMeta(unittest2.TestCase): + def test_configdrive_scheme(self, mock_lbd, mock_image_meta): mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE cd_scheme = nailgun.Nailgun(PROVISION_SAMPLE_DATA).configdrive_scheme self.assertEqual(['fake_authorized_key1', 'fake_authorized_key2', @@ -876,10 +942,7 @@ class TestNailgun(unittest2.TestCase): ], cd_scheme.common.ks_repos) - @mock.patch('yaml.load') - @mock.patch.object(utils, 'init_http_request') - @mock.patch.object(hu, 'list_block_devices') - def test_partition_scheme(self, mock_lbd, mock_http_req, mock_yaml): + def test_partition_scheme(self, mock_lbd, mock_image_meta): mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE drv = nailgun.Nailgun(PROVISION_SAMPLE_DATA) p_scheme = drv.partition_scheme @@ -889,10 +952,7 @@ class TestNailgun(unittest2.TestCase): self.assertEqual(2, len(p_scheme.vgs)) self.assertEqual(3, len(p_scheme.parteds)) - @mock.patch('yaml.load') - @mock.patch.object(utils, 'init_http_request') - @mock.patch.object(hu, 'list_block_devices') - def test_image_scheme(self, mock_lbd, mock_http_req, mock_yaml): + def test_image_scheme(self, mock_lbd, mock_image_meta): mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE drv = nailgun.Nailgun(PROVISION_SAMPLE_DATA) p_scheme = drv.partition_scheme @@ -920,20 +980,16 @@ class TestNailgun(unittest2.TestCase): self.assertIsNone(img.size) self.assertIsNone(img.md5) - @mock.patch.object(utils, 'init_http_request') - @mock.patch.object(hu, 'list_block_devices') - def test_image_scheme_with_checksums(self, mock_lbd, mock_http_req): - fake_image_meta = {'images': [{'raw_md5': 'fakeroot', 'raw_size': 1, - 'container_name': 'fake_image.img.gz'}]} - prop_mock = mock.PropertyMock(return_value=yaml.dump(fake_image_meta)) - type(mock_http_req.return_value).text = prop_mock + def test_image_scheme_with_checksums(self, mock_lbd, mock_image_meta): + fake_image_meta = { + 'images': [{'raw_md5': 'fakeroot', 'raw_size': 1, + 'container_name': 'fake_image.img.gz'}]} + mock_image_meta.return_value = fake_image_meta mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE p_data = PROVISION_SAMPLE_DATA.copy() drv = nailgun.Nailgun(p_data) p_scheme = drv.partition_scheme i_scheme = drv.image_scheme - mock_http_req.assert_called_once_with( - 'http://fake.host.org:123/imgs/fake_image.yaml') expected_images = [] for fs in p_scheme.fss: if fs.mount not in PROVISION_SAMPLE_DATA['ks_meta']['image_data']: @@ -958,10 +1014,7 @@ class TestNailgun(unittest2.TestCase): img.size, fake_image_meta['images'][0]['raw_size']) self.assertEqual(img.md5, fake_image_meta['images'][0]['raw_md5']) - @mock.patch('yaml.load') - @mock.patch.object(utils, 'init_http_request') - @mock.patch.object(hu, 'list_block_devices') - def test_getlabel(self, mock_lbd, mock_http_req, mock_yaml): + def test_getlabel(self, mock_lbd, mock_image_meta): mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE drv = nailgun.Nailgun(PROVISION_SAMPLE_DATA) self.assertEqual('', drv._getlabel(None)) @@ -969,10 +1022,7 @@ class TestNailgun(unittest2.TestCase): self.assertEqual(' -L %s ' % long_label[:12], drv._getlabel(long_label)) - @mock.patch('yaml.load') - @mock.patch.object(utils, 'init_http_request') - @mock.patch.object(hu, 'list_block_devices') - def test_disk_dev_not_found(self, mock_lbd, mock_http_req, mock_yaml): + def test_disk_dev_not_found(self, mock_lbd, mock_image_meta): mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE drv = nailgun.Nailgun(PROVISION_SAMPLE_DATA) fake_ks_disk = { @@ -985,19 +1035,13 @@ class TestNailgun(unittest2.TestCase): self.assertRaises(errors.DiskNotFoundError, drv._disk_dev, fake_ks_disk) - @mock.patch('yaml.load') - @mock.patch.object(utils, 'init_http_request') - @mock.patch.object(hu, 'list_block_devices') - def test_get_partition_count(self, mock_lbd, mock_http_req, mock_yaml): + def test_get_partition_count(self, mock_lbd, mock_image_meta): mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE drv = nailgun.Nailgun(PROVISION_SAMPLE_DATA) self.assertEqual(3, drv._get_partition_count('Boot')) self.assertEqual(1, drv._get_partition_count('TMP')) - @mock.patch('yaml.load') - @mock.patch.object(utils, 'init_http_request') - @mock.patch.object(hu, 'list_block_devices') - def test_partition_scheme_ceph(self, mock_lbd, mock_http_req, mock_yaml): + def test_partition_scheme_ceph(self, mock_lbd, mock_image_meta): # TODO(agordeev): perform better testing of ceph logic p_data = copy.deepcopy(PROVISION_SAMPLE_DATA) for i in range(0, 3): @@ -1020,10 +1064,7 @@ class TestNailgun(unittest2.TestCase): self.assertEqual(CEPH_DATA['partition_guid'], p_scheme.parteds[disk].partitions[part].guid) - @mock.patch('fuel_agent.drivers.nailgun.yaml.load') - @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request') - @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices') - def test_grub_centos_26(self, mock_lbd, mock_http_req, mock_yaml): + def test_grub_centos_26(self, mock_lbd, mock_image_meta): data = copy.deepcopy(PROVISION_SAMPLE_DATA) data['profile'] = 'centos' data['ks_meta']['kernel_lt'] = 0 @@ -1033,14 +1074,11 @@ class TestNailgun(unittest2.TestCase): ' ' + data['ks_meta']['pm_data']['kernel_params']) self.assertEqual(drv.grub.kernel_regexp, r'^vmlinuz-2\.6.*') self.assertEqual(drv.grub.initrd_regexp, r'^initramfs-2\.6.*') - self.assertIsNone(drv.grub.version) + self.assertEqual(1, drv.grub.version) self.assertIsNone(drv.grub.kernel_name) self.assertIsNone(drv.grub.initrd_name) - @mock.patch('fuel_agent.drivers.nailgun.yaml.load') - @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request') - @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices') - def test_grub_centos_lt(self, mock_lbd, mock_http_req, mock_yaml): + def test_grub_centos_lt(self, mock_lbd, mock_image_meta): data = copy.deepcopy(PROVISION_SAMPLE_DATA) data['profile'] = 'centos' data['ks_meta']['kernel_lt'] = 1 @@ -1050,14 +1088,11 @@ class TestNailgun(unittest2.TestCase): ' ' + data['ks_meta']['pm_data']['kernel_params']) self.assertIsNone(drv.grub.kernel_regexp) self.assertIsNone(drv.grub.initrd_regexp) - self.assertIsNone(drv.grub.version) + self.assertEqual(1, drv.grub.version) self.assertIsNone(drv.grub.kernel_name) self.assertIsNone(drv.grub.initrd_name) - @mock.patch('fuel_agent.drivers.nailgun.yaml.load') - @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request') - @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices') - def test_grub_ubuntu(self, mock_lbd, mock_http_req, mock_yaml): + def test_grub_ubuntu(self, mock_lbd, mock_image_meta): data = copy.deepcopy(PROVISION_SAMPLE_DATA) data['profile'] = 'ubuntu' data['ks_meta']['kernel_lt'] = 0 @@ -1065,17 +1100,13 @@ class TestNailgun(unittest2.TestCase): drv = nailgun.Nailgun(data) self.assertEqual(drv.grub.kernel_params, ' ' + data['ks_meta']['pm_data']['kernel_params']) - self.assertIsNone(drv.grub.version) + self.assertEqual(2, drv.grub.version) self.assertIsNone(drv.grub.kernel_regexp) self.assertIsNone(drv.grub.initrd_regexp) self.assertIsNone(drv.grub.kernel_name) self.assertIsNone(drv.grub.initrd_name) - @mock.patch('fuel_agent.drivers.nailgun.yaml.load') - @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request') - @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices') - def test_boot_partition_ok_single_disk(self, mock_lbd, - mock_http_req, mock_yaml): + def test_boot_partition_ok_single_disk(self, mock_lbd, mock_image_meta): data = copy.deepcopy(PROVISION_SAMPLE_DATA) data['ks_meta']['pm_data']['ks_spaces'] = SINGLE_DISK_KS_SPACES mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE @@ -1084,11 +1115,7 @@ class TestNailgun(unittest2.TestCase): drv.partition_scheme.fs_by_mount('/boot').device, '/dev/sda3') - @mock.patch('fuel_agent.drivers.nailgun.yaml.load') - @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request') - @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices') - def test_elevate_keep_data_single_disk(self, mock_lbd, - mock_http_req, mock_yaml): + def test_elevate_keep_data_single_disk(self, mock_lbd, mock_image_meta): data = copy.deepcopy(PROVISION_SAMPLE_DATA) data['ks_meta']['pm_data']['ks_spaces'] = SINGLE_DISK_KS_SPACES mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE @@ -1115,11 +1142,8 @@ class TestNailgun(unittest2.TestCase): if fs.mount != '/': self.assertFalse(fs.keep_data) - @mock.patch('fuel_agent.drivers.nailgun.yaml.load') - @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request') - @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices') def test_boot_partition_ok_many_normal_disks(self, mock_lbd, - mock_http_req, mock_yaml): + mock_image_meta): data = copy.deepcopy(PROVISION_SAMPLE_DATA) mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE drv = nailgun.Nailgun(data) @@ -1127,11 +1151,8 @@ class TestNailgun(unittest2.TestCase): drv.partition_scheme.fs_by_mount('/boot').device, '/dev/sda3') - @mock.patch('fuel_agent.drivers.nailgun.yaml.load') - @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request') - @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices') def test_boot_partition_ok_first_disk_huge(self, mock_lbd, - mock_http_req, mock_yaml): + mock_image_meta): data = copy.deepcopy(PROVISION_SAMPLE_DATA) data['ks_meta']['pm_data']['ks_spaces'] = FIRST_DISK_HUGE_KS_SPACES mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE @@ -1140,11 +1161,8 @@ class TestNailgun(unittest2.TestCase): drv.partition_scheme.fs_by_mount('/boot').device, '/dev/sdb3') - @mock.patch('fuel_agent.drivers.nailgun.yaml.load') - @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request') - @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices') def test_boot_partition_ok_many_huge_disks(self, mock_lbd, - mock_http_req, mock_yaml): + mock_image_meta): data = copy.deepcopy(PROVISION_SAMPLE_DATA) data['ks_meta']['pm_data']['ks_spaces'] = MANY_HUGE_DISKS_KS_SPACES mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE @@ -1153,22 +1171,14 @@ class TestNailgun(unittest2.TestCase): drv.partition_scheme.fs_by_mount('/boot').device, '/dev/sda3') - @mock.patch('fuel_agent.drivers.nailgun.yaml.load') - @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request') - @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices') - def test_boot_partition_no_boot(self, mock_lbd, - mock_http_req, mock_yaml): + def test_boot_partition_no_boot(self, mock_lbd, mock_image_meta): data = copy.deepcopy(PROVISION_SAMPLE_DATA) data['ks_meta']['pm_data']['ks_spaces'] = NO_BOOT_KS_SPACES mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE self.assertRaises(errors.WrongPartitionSchemeError, nailgun.Nailgun, data) - @mock.patch('fuel_agent.drivers.nailgun.yaml.load') - @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request') - @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices') - def test_boot_partition_no_boot_nvme(self, mock_lbd, - mock_http_req, mock_yaml): + def test_boot_partition_no_boot_nvme(self, mock_lbd, mock_image_meta): data = copy.deepcopy(PROVISION_SAMPLE_DATA) data['ks_meta']['pm_data']['ks_spaces'] = ONLY_ONE_NVME_KS_SPACES mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE_NVME @@ -1177,11 +1187,7 @@ class TestNailgun(unittest2.TestCase): '/boot partition has not been created for some reasons'): nailgun.Nailgun(data) - @mock.patch('fuel_agent.drivers.nailgun.yaml.load') - @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request') - @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices') - def test_boot_partition_is_not_on_nvme(self, mock_lbd, - mock_http_req, mock_yaml): + def test_boot_partition_is_not_on_nvme(self, mock_lbd, mock_image_meta): data = copy.deepcopy(PROVISION_SAMPLE_DATA) data['ks_meta']['pm_data']['ks_spaces'] = FIRST_DISK_NVME_KS_SPACES mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE_NVME @@ -1189,3 +1195,58 @@ class TestNailgun(unittest2.TestCase): self.assertEqual( drv.partition_scheme.fs_by_mount('/boot').device, '/dev/sda3') + + def test_md_metadata_centos(self, mock_lbd, mock_image_meta): + data = copy.deepcopy(PROVISION_SAMPLE_DATA) + data['profile'] = 'base-centos-x86_64' + data['ks_meta']['pm_data']['ks_spaces'] = MD_RAID_KS_SPACES + mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE + drv = nailgun.Nailgun(data) + self.assertEqual(1, drv.grub.version) + self.assertEqual(1, len(drv.partition_scheme.mds)) + self.assertEqual('0.90', drv.partition_scheme.mds[0].metadata) + + def test_md_metadata_centos70(self, mock_lbd, mock_image_meta): + data = copy.deepcopy(PROVISION_SAMPLE_DATA) + data['profile'] = 'base-centos7-x86_64' + data['ks_meta']['pm_data']['ks_spaces'] = MD_RAID_KS_SPACES + mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE + drv = nailgun.Nailgun(data) + self.assertEqual(2, drv.grub.version) + self.assertEqual(1, len(drv.partition_scheme.mds)) + self.assertEqual('default', drv.partition_scheme.mds[0].metadata) + + def test_md_metadata_ubuntu(self, mock_lbd, mock_image_meta): + data = copy.deepcopy(PROVISION_SAMPLE_DATA) + data['profile'] = 'base-ubuntu_1404_x86_64' + data['ks_meta']['pm_data']['ks_spaces'] = MD_RAID_KS_SPACES + mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE + drv = nailgun.Nailgun(data) + self.assertEqual(1, len(drv.partition_scheme.mds)) + self.assertEqual(2, drv.grub.version) + self.assertEqual('default', drv.partition_scheme.mds[0].metadata) + + +@mock.patch.object(utils, 'init_http_request') +@mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices') +class TestNailgunImageMeta(unittest2.TestCase): + def test_parse_image_meta(self, mock_lbd, mock_http_req): + fake_image_meta = {'images': [{'raw_md5': 'fakeroot', 'raw_size': 1, + 'container_name': 'fake_image.img.gz'}]} + prop_mock = mock.PropertyMock(return_value=yaml.dump(fake_image_meta)) + type(mock_http_req.return_value).text = prop_mock + mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE + p_data = PROVISION_SAMPLE_DATA.copy() + drv = nailgun.Nailgun(p_data) + self.assertEqual(fake_image_meta, drv._image_meta) + mock_http_req.assert_called_once_with( + 'http://fake.host.org:123/imgs/fake_image.yaml') + + def test_parse_image_meta_not_parsed(self, mock_lbd, mock_http_req): + mock_http_req.side_effect = KeyError() + mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE + p_data = PROVISION_SAMPLE_DATA.copy() + drv = nailgun.Nailgun(p_data) + self.assertEqual({}, drv._image_meta) + mock_http_req.assert_called_once_with( + 'http://fake.host.org:123/imgs/fake_image.yaml') diff --git a/fuel_agent/tests/test_nailgun_build_image.py b/fuel_agent/tests/test_nailgun_build_image.py index 1dfbb09..7f53c3a 100644 --- a/fuel_agent/tests/test_nailgun_build_image.py +++ b/fuel_agent/tests/test_nailgun_build_image.py @@ -125,7 +125,8 @@ class TestNailgunBuildImage(unittest2.TestCase): mock_ub_instance = mock_ub.return_value mock_ub_instance.packages = data['packages'] driver = NailgunBuildImage(data) - mock_ub.assert_called_once_with(repos=[], packages=data['packages']) + mock_ub.assert_called_once_with(repos=[], packages=data['packages'], + major=14, minor=4) self.assertEqual(driver.operating_system.packages, data['packages']) @mock.patch('fuel_agent.objects.Ubuntu') @@ -140,7 +141,8 @@ class TestNailgunBuildImage(unittest2.TestCase): mock_ub_instance.packages = NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES driver = NailgunBuildImage(data) mock_ub.assert_called_once_with( - repos=[], packages=NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES) + repos=[], packages=NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES, + major=14, minor=4) self.assertEqual(driver.operating_system.packages, NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES) @@ -170,7 +172,8 @@ class TestNailgunBuildImage(unittest2.TestCase): mock_ub_instance = mock_ub.return_value mock_ub_instance.repos = repos mock_ub.assert_called_once_with( - repos=repos, packages=NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES) + repos=repos, packages=NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES, + major=14, minor=4) self.assertEqual(mock_deb_expected_calls, mock_deb.call_args_list[:len(REPOS_SAMPLE)]) self.assertEqual(driver.operating_system.repos, repos) diff --git a/fuel_agent/tests/test_partition.py b/fuel_agent/tests/test_partition.py index 678f1b4..b6f042c 100644 --- a/fuel_agent/tests/test_partition.py +++ b/fuel_agent/tests/test_partition.py @@ -67,6 +67,7 @@ class TestMultipleDevice(unittest2.TestCase): 'level': 'level', 'devices': ['device_a', ], 'spares': ['device_b', ], + 'metadata': 'default', 'keep_data': False, } new_md = objects.MultipleDevice.from_dict(serialized) diff --git a/fuel_agent/tests/test_simple_nailgun_driver.py b/fuel_agent/tests/test_simple_nailgun_driver.py index 94e1838..511d776 100644 --- a/fuel_agent/tests/test_simple_nailgun_driver.py +++ b/fuel_agent/tests/test_simple_nailgun_driver.py @@ -24,6 +24,8 @@ from fuel_agent.tests import base @mock.patch.multiple( simple.NailgunSimpleDriver, + parse_operating_system=lambda x: objects.OperatingSystem(None, None), + parse_image_meta=lambda x: {}, parse_grub=lambda x: objects.Grub(), parse_configdrive_scheme=lambda x: objects.ConfigDriveScheme(), parse_image_scheme=lambda x: objects.ImageScheme()) diff --git a/fuel_agent/utils/md.py b/fuel_agent/utils/md.py index f9b1076..b890573 100644 --- a/fuel_agent/utils/md.py +++ b/fuel_agent/utils/md.py @@ -76,7 +76,7 @@ def mddisplay(names=None): return mds -def mdcreate(mdname, level, devices): +def mdcreate(mdname, level, devices, metadata='default'): mds = mddisplay() # check if md device already exists @@ -107,7 +107,7 @@ def mdcreate(mdname, level, devices): # FIXME: mdadm will ask user to continue creating if any device appears to # be a part of raid array. Superblock zeroing helps to avoid that. map(mdclean, devices) - utils.execute('mdadm', '--create', '--force', mdname, '-e0.90', + utils.execute('mdadm', '--create', '--force', mdname, '-e', metadata, '--level=%s' % level, '--raid-devices=%s' % len(devices), *devices, check_exit_code=[0])