Merge "fuel-agent: use different md metadata version"

fuel_agent/drivers/nailgun.py

@@ -80,9 +80,13 @@ class Nailgun(BaseDataDriver):
         # has already been added. we need this to
         # get rid of md over all disks for /boot partition.
         self._boot_done = False
+        self._image_meta = self.parse_image_meta()
 
-        self._partition_scheme = self.parse_partition_scheme()
+        self._operating_system = self.parse_operating_system()
         self._grub = self.parse_grub()
+        # parsing partition scheme needs grub and operating system have
+        # been parsed
+        self._partition_scheme = self.parse_partition_scheme()
         self._configdrive_scheme = self.parse_configdrive_scheme()
         # parsing image scheme needs partition scheme has been parsed
         self._image_scheme = self.parse_image_scheme()
@@ -99,9 +103,14 @@ class Nailgun(BaseDataDriver):
     def grub(self):
         return self._grub
 
+    @property
+    def have_grub1_by_default(self):
+        return (isinstance(self.operating_system, objects.Centos) and
+                self.operating_system.major == 6)
+
     @property
     def operating_system(self):
-        return None
+        return self._operating_system
 
     @property
     def configdrive_scheme(self):
@@ -174,6 +183,49 @@ class Nailgun(BaseDataDriver):
     def _num_ceph_osds(self):
         return self._get_partition_count('ceph')
 
+    def get_os_by_image_meta(self, os_release):
+        LOG.debug('--- Getting operating system data by image metadata ---')
+        if os_release:
+            LOG.debug('Looks like %{0} is going to be provisioned'.
+                      format(os_release))
+            try:
+                OS = getattr(objects, os_release['name'])
+                os = OS(repos=None, packages=None, major=os_release['major'],
+                        minor=os_release['minor'])
+                return os
+            except (AttributeError, KeyError):
+                LOG.warning('Cannot guess operating system release '
+                            'from image metadata')
+
+    def get_os_by_profile(self, profile):
+        LOG.debug('--- Getting operating system data by profile ---')
+        if 'centos' in profile:
+            os = objects.Centos(repos=None, packages=None, major=6, minor=5)
+            if '7' in profile:
+                LOG.debug('Looks like CentOS7.0 is going to be provisioned.')
+                os = objects.Centos(repos=None, packages=None, major=7,
+                                    minor=0)
+            else:
+                LOG.debug('Looks like CentOS6.5 is going to be provisioned.')
+            return os
+        elif 'ubuntu' in profile:
+            os = objects.Ubuntu(repos=None, packages=None, major=12, minor=4)
+            if '1404' in profile:
+                LOG.debug('Looks like Ubuntu1404 is going to be provisioned.')
+                os = objects.Ubuntu(repos=None, packages=None, major=14,
+                                    minor=4)
+            else:
+                LOG.debug('Looks like Ubuntu1204 is going to be provisioned.')
+            return os
+        os = objects.OperatingSystem(repos=None, packages=None)
+        return os
+
+    def parse_operating_system(self):
+        LOG.debug('--- Preparing operating system data ---')
+        os_release = self._image_meta.get('os', None)
+        return self.get_os_by_image_meta(os_release) or \
+            self.get_os_by_profile(self.data['profile'].lower())
+
     def parse_partition_scheme(self):
         LOG.debug('--- Preparing partition scheme ---')
         data = self.partition_data()
@@ -344,10 +396,18 @@ class Nailgun(BaseDataDriver):
                             volume['mount'] not in ('none', '/boot'):
                         LOG.debug('Attaching partition to RAID '
                                   'by its mount point %s' % volume['mount'])
+                        metadata = 'default'
+                        if self.have_grub1_by_default:
+                            metadata = '0.90'
+                        LOG.debug('Going to use MD metadata version {0}. '
+                                  'The version was guessed at the data has '
+                                  'been given about the operating system.'
+                                  .format(metadata))
                         partition_scheme.md_attach_by_mount(
                             device=prt.name, mount=volume['mount'],
                             fs_type=volume.get('file_system', 'xfs'),
-                            fs_label=self._getlabel(volume.get('disk_label')))
+                            fs_label=self._getlabel(volume.get('disk_label')),
+                            metadata=metadata)
 
                     if 'mount' in volume and volume['mount'] == '/boot' and \
                             not self._boot_done:
@@ -469,12 +529,13 @@ class Nailgun(BaseDataDriver):
             LOG.debug('Prefered kernel version is 2.6')
             grub.kernel_regexp = r'^vmlinuz-2\.6.*'
             grub.initrd_regexp = r'^initramfs-2\.6.*'
+        grub.version = 1 if self.have_grub1_by_default else 2
+        LOG.debug('Grub version is %{0}'.format(grub.version))
         return grub
 
-    def parse_image_scheme(self):
+    def parse_image_meta(self):
-        LOG.debug('--- Preparing image scheme ---')
+        LOG.debug('--- Preparing image metadata ---')
         data = self.data
-        image_scheme = objects.ImageScheme()
         # FIXME(agordeev): this piece of code for fetching additional image
         # meta data should be factored out of this particular nailgun driver
         # into more common and absract data getter which should be able to deal
@@ -496,6 +557,13 @@ class Nailgun(BaseDataDriver):
             LOG.exception(e)
             LOG.debug('Failed to fetch/decode image meta data')
             image_meta = {}
+        return image_meta
+
+    def parse_image_scheme(self):
+        LOG.debug('--- Preparing image scheme ---')
+        data = self.data
+        image_meta = self._image_meta
+        image_scheme = objects.ImageScheme()
         # We assume for every file system user may provide a separate
         # file system image. For example if partitioning scheme has
         # /, /boot, /var/lib file systems then we will try to get images
@@ -619,8 +687,8 @@ class NailgunBuildImage(BaseDataDriver):
                 suite=repo['suite'],
                 section=repo['section'],
                 priority=repo['priority']))
-
+        os = objects.Ubuntu(repos=repos, packages=packages, major=14, minor=4)
-        return objects.Ubuntu(repos=repos, packages=packages)
+        return os
 
     def parse_schemes(self):
 

fuel_agent/manager.py

@@ -181,7 +181,7 @@ class Manager(object):
 
         # creating meta disks
         for md in self.driver.partition_scheme.mds:
-            mu.mdcreate(md.name, md.level, md.devices)
+            mu.mdcreate(md.name, md.level, md.devices, md.metadata)
 
         # creating physical volumes
         for pv in self.driver.partition_scheme.pvs:
@@ -362,7 +362,12 @@ class Manager(object):
 
         grub = self.driver.grub
 
-        grub.version = gu.guess_grub_version(chroot=chroot)
+        guessed_version = gu.guess_grub_version(chroot=chroot)
+        if guessed_version != grub.version:
+            grub.version = guessed_version
+            LOG.warning('Grub version differs from which the operating system '
+                        'should have by default. Found version in image: '
+                        '{0}'.format(guessed_version))
         boot_device = self.driver.partition_scheme.boot_device(grub.version)
         install_devices = [d.name for d in self.driver.partition_scheme.parteds
                            if d.install_bootloader]
@@ -471,6 +476,8 @@ class Manager(object):
         # as a pluggable data driver to avoid any fixed format.
         metadata = {}
 
+        metadata['os'] = self.driver.operating_system.to_dict()
+
         # TODO(kozhukalov): implement this using image metadata
         # we need to compare list of packages and repos
         LOG.info('*** Checking if image exists ***')

fuel_agent/objects/__init__.py

@@ -19,6 +19,7 @@ from fuel_agent.objects.configdrive import ConfigDriveScheme
 from fuel_agent.objects.device import Loop
 from fuel_agent.objects.image import Image
 from fuel_agent.objects.image import ImageScheme
+from fuel_agent.objects.operating_system import Centos
 from fuel_agent.objects.operating_system import OperatingSystem
 from fuel_agent.objects.operating_system import Ubuntu
 from fuel_agent.objects.partition.fs import FileSystem
@@ -63,6 +64,7 @@ __all__ = [
     'Grub',
     'OperatingSystem',
     'Ubuntu',
+    'Centos',
     'Repo',
     'DEBRepo',
     'Loop',

fuel_agent/objects/operating_system.py

@@ -14,10 +14,21 @@
 
 
 class OperatingSystem(object):
-    def __init__(self, repos, packages):
+    def __init__(self, repos, packages, major='unknown', minor='unknown'):
         self.repos = repos
         self.packages = packages
+        self.major = major
+        self.minor = minor
+
+    def to_dict(self):
+        return {'major': self.major,
+                'minor': self.minor,
+                'name': self.__class__.__name__}
 
 
 class Ubuntu(OperatingSystem):
     pass
+
+
+class Centos(OperatingSystem):
+    pass

fuel_agent/objects/partition/md.py

@@ -20,12 +20,14 @@ from fuel_agent.objects import base
 class MultipleDevice(base.Serializable):
 
     def __init__(self, name, level,
-                 devices=None, spares=None, keep_data=False):
+                 devices=None, spares=None, keep_data=False,
+                 metadata='default'):
         self.keep_data = keep_data
         self.name = name
         self.level = level
         self.devices = devices or []
         self.spares = spares or []
+        self.metadata = metadata
 
     def add_device(self, device):
         if device in self.devices or device in self.spares:
@@ -48,4 +50,5 @@ class MultipleDevice(base.Serializable):
             'devices': self.devices,
             'spares': self.spares,
             'keep_data': self.keep_data,
+            'metadata': self.metadata,
         }

fuel_agent/objects/partition/scheme.py

@@ -67,6 +67,7 @@ class PartitionScheme(object):
         mdkwargs = {}
         mdkwargs['name'] = kwargs.get('name') or self.md_next_name()
         mdkwargs['level'] = kwargs.get('level') or 'mirror'
+        mdkwargs['metadata'] = kwargs.get('metadata') or 'default'
         md = MultipleDevice(**mdkwargs)
         self.mds.append(md)
         return md

fuel_agent/tests/test_manager.py

@@ -38,10 +38,10 @@ CONF = cfg.CONF
 
 class TestManager(unittest2.TestCase):
 
-    @mock.patch('yaml.load')
+    @mock.patch('fuel_agent.drivers.nailgun.Nailgun.parse_image_meta',
-    @mock.patch.object(utils, 'init_http_request')
+                return_value={})
     @mock.patch.object(hu, 'list_block_devices')
-    def setUp(self, mock_lbd, mock_http, mock_yaml):
+    def setUp(self, mock_lbd, mock_image_meta):
         super(TestManager, self).setUp()
         mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
         self.mgr = manager.Manager(test_nailgun.PROVISION_SAMPLE_DATA)
@@ -123,6 +123,7 @@ class TestManager(unittest2.TestCase):
             else:
                 return ('FAKE_UUID', None)
         mock_utils.execute.side_effect = _fake_uuid
+        mock_grub.version = 2
         mock_gu.guess_grub_version.return_value = 2
         mock_grub.kernel_name = 'fake_kernel_name'
         mock_grub.initrd_name = 'fake_initrd_name'
@@ -145,6 +146,22 @@ class TestManager(unittest2.TestCase):
         self.assertRaises(errors.WrongPartitionSchemeError,
                           self.mgr.do_bootloader)
 
+    @mock.patch('fuel_agent.manager.open',
+                create=True, new_callable=mock.mock_open)
+    @mock.patch('fuel_agent.manager.gu', create=True)
+    @mock.patch('fuel_agent.manager.utils', create=True)
+    @mock.patch.object(manager.Manager, 'mount_target')
+    @mock.patch.object(manager.Manager, 'umount_target')
+    def test_do_bootloader_grub_version_changes(
+            self, mock_umount, mock_mount, mock_utils, mock_gu, mock_open):
+        # actually covers only grub1 related logic
+        mock_utils.execute.return_value = ('fake_UUID\n', None)
+        mock_gu.guess_grub_version.return_value = 'expected_version'
+        self.mgr.do_bootloader()
+        mock_gu.guess_grub_version.assert_called_once_with(
+            chroot='/tmp/target')
+        self.assertEqual('expected_version', self.mgr.driver.grub.version)
+
     @mock.patch('fuel_agent.manager.open',
                 create=True, new_callable=mock.mock_open)
     @mock.patch('fuel_agent.manager.gu', create=True)
@@ -245,12 +262,12 @@ class TestManager(unittest2.TestCase):
         mock_utils.makedirs_if_not_exists.assert_called_once_with(
             '/tmp/target/etc/nailgun-agent')
 
-    @mock.patch('yaml.load')
+    @mock.patch('fuel_agent.drivers.nailgun.Nailgun.parse_image_meta',
-    @mock.patch.object(utils, 'init_http_request')
+                return_value={})
     @mock.patch.object(hu, 'list_block_devices')
     @mock.patch.object(fu, 'make_fs')
     def test_do_partitioning_with_keep_data_flag(self, mock_fu_mf, mock_lbd,
-                                                 mock_http, mock_yaml):
+                                                 mock_image_meta):
         mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
         data = copy.deepcopy(test_nailgun.PROVISION_SAMPLE_DATA)
 
@@ -301,9 +318,9 @@ class TestManager(unittest2.TestCase):
         ]
         self.mgr.do_partitioning()
         self.assertEqual([mock.call('fake_md1', 'mirror',
-                                    ['/dev/sda1', '/dev/sdb1']),
+                                    ['/dev/sda1', '/dev/sdb1'], 'default'),
                           mock.call('fake_md2', 'mirror',
-                                    ['/dev/sdb3', '/dev/sdc1'])],
+                                    ['/dev/sdb3', '/dev/sdc1'], 'default')],
                          mock_mu_m.call_args_list)
 
     @mock.patch.object(manager.os.path, 'exists')
@@ -393,6 +410,9 @@ class TestManager(unittest2.TestCase):
             mock.call('xfs', '', '', '/dev/mapper/image-glance')]
         self.assertEqual(mock_fu_mf_expected_calls, mock_fu_mf.call_args_list)
 
+    @mock.patch('fuel_agent.drivers.nailgun.Nailgun.parse_image_meta',
+                return_value={})
+    @mock.patch('fuel_agent.drivers.nailgun.Nailgun.parse_operating_system')
     @mock.patch.object(utils, 'calculate_md5')
     @mock.patch('os.path.getsize')
     @mock.patch('yaml.load')
@@ -401,7 +421,8 @@ class TestManager(unittest2.TestCase):
     @mock.patch.object(utils, 'render_and_save')
     @mock.patch.object(hu, 'list_block_devices')
     def test_do_configdrive(self, mock_lbd, mock_u_ras, mock_u_e,
-                            mock_http_req, mock_yaml, mock_get_size, mock_md5):
+                            mock_http_req, mock_yaml, mock_get_size, mock_md5,
+                            mock_parse_os, mock_image_meta):
         mock_get_size.return_value = 123
         mock_md5.return_value = 'fakemd5'
         mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
@@ -753,7 +774,8 @@ class TestImageBuild(unittest2.TestCase):
                 objects.DEBRepo('mos', 'http://fakemos',
                                 'mosX.Y', 'fakesection', priority=1000)],
             packages=['fakepackage1', 'fakepackage2'])
-
+        self.mgr.driver.operating_system.minor = 4
+        self.mgr.driver.operating_system.major = 14
         mock_os.path.exists.return_value = False
         mock_os.path.join.return_value = '/tmp/imgdir/proc'
         mock_os.path.basename.side_effect = ['img.img.gz', 'img-boot.img.gz']
@@ -879,7 +901,7 @@ class TestImageBuild(unittest2.TestCase):
              mock.call('/tmp/img-boot.gz', '/fake/img-boot.img.gz')],
             mock_shutil_move.call_args_list)
 
-        metadata = {}
+        metadata = {'os': {'name': 'Ubuntu', 'major': 14, 'minor': 4}}
         for repo in self.mgr.driver.operating_system.repos:
             metadata.setdefault('repos', []).append({
                 'type': 'deb',

fuel_agent/tests/test_md_utils.py

@@ -159,7 +159,7 @@ localhost.localdomain)
         self.assertEqual(mock_mdclean_expected_calls,
                          mock_mdclean.call_args_list)
         mock_exec.assert_called_once_with(
-            'mdadm', '--create', '--force', '/dev/md0', '-e0.90',
+            'mdadm', '--create', '--force', '/dev/md0', '-e', 'default',
             '--level=mirror',
             '--raid-devices=2', '/dev/fake1', '/dev/fake2',
             check_exit_code=[0])

fuel_agent/tests/test_nailgun.py

@@ -20,8 +20,8 @@ import yaml
 
 from fuel_agent.drivers import nailgun
 from fuel_agent import errors
+from fuel_agent import objects
 from fuel_agent.objects import image
-from fuel_agent.utils import hardware as hu
 from fuel_agent.utils import utils
 
 
@@ -502,6 +502,37 @@ NO_BOOT_KS_SPACES = [
     }
 ]
 
+MD_RAID_KS_SPACES = [
+    {
+        "name": "sda",
+        "extra": ["sda"],
+        "free_space": 1024,
+        "volumes": [
+            {
+                "type": "boot",
+                "size": 300
+            },
+            {
+                "mount": "/boot",
+                "size": 200,
+                "type": "raid",
+                "file_system": "ext2",
+                "name": "Boot"
+            },
+            {
+                "mount": "/",
+                "size": 200,
+                "type": "raid",
+                "file_system": "ext4",
+                "name": "Root"
+            },
+        ],
+        "type": "disk",
+        "id": "sda",
+        "size": 102400
+    }
+]
+
 FIRST_DISK_HUGE_KS_SPACES = [
     {
         "name": "sda",
@@ -711,8 +742,7 @@ MANY_HUGE_DISKS_KS_SPACES = [
 ]
 
 
-class TestNailgun(unittest2.TestCase):
+class TestNailgunMatch(unittest2.TestCase):
-
     def test_match_device_by_id_matches(self):
         # matches by 'by-id' links
         fake_ks_disk = {
@@ -824,10 +854,46 @@ class TestNailgun(unittest2.TestCase):
         }
         self.assertFalse(nailgun.match_device(fake_hu_disk, fake_ks_disk))
 
-    @mock.patch('yaml.load')
+
-    @mock.patch.object(utils, 'init_http_request')
+@mock.patch.object(nailgun.Nailgun, '__init__', return_value=None)
-    @mock.patch.object(hu, 'list_block_devices')
+class TestNailgunGetOSMethods(unittest2.TestCase):
-    def test_configdrive_scheme(self, mock_lbd, mock_http, mock_yaml):
+    def test_parse_operating_system_test_profiles(self, mock_nailgun):
+        d = {'centos-x86_64': {'obj': objects.Centos, 'minor': 5, 'major': 6},
+             'centos7-x86_64': {'obj': objects.Centos, 'minor': 0, 'major': 7},
+             'ubuntu_1204_x86_64': {'obj': objects.Ubuntu,
+                                    'minor': 4, 'major': 12},
+             'ubuntu_1404_x86_64': {'obj': objects.Ubuntu,
+                                    'minor': 4, 'major': 14},
+             'generic_os': {'obj': objects.OperatingSystem,
+                            'minor': 'unknown', 'major': 'unknown'}}
+        drv = nailgun.Nailgun('fake_data')
+        for profile, obj in d.iteritems():
+            os = drv.get_os_by_profile(profile)
+            self.assertIsInstance(os, obj['obj'])
+            self.assertEqual(obj['minor'], os.minor)
+            self.assertEqual(obj['major'], os.major)
+
+    def test_parse_operating_system_image_meta(self, mock_nailgun):
+        d = {'Centos': objects.Centos,
+             'Ubuntu': objects.Ubuntu,
+             'unknown': None}
+        drv = nailgun.Nailgun('fake_data')
+        for os_name, obj in d.iteritems():
+            os = drv.get_os_by_image_meta(
+                {'name': os_name, 'minor': 1, 'major': 2})
+            if os:
+                self.assertIsInstance(os, obj)
+                self.assertEqual(1, os.minor)
+                self.assertEqual(2, os.major)
+            else:
+                self.assertIsNone(os)
+                self.assertEqual('unknown', os_name)
+
+
+@mock.patch.object(nailgun.Nailgun, 'parse_image_meta', return_value={})
+@mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices')
+class TestNailgunMockedMeta(unittest2.TestCase):
+    def test_configdrive_scheme(self, mock_lbd, mock_image_meta):
         mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
         cd_scheme = nailgun.Nailgun(PROVISION_SAMPLE_DATA).configdrive_scheme
         self.assertEqual(['fake_authorized_key1', 'fake_authorized_key2',
@@ -876,10 +942,7 @@ class TestNailgun(unittest2.TestCase):
             ],
             cd_scheme.common.ks_repos)
 
-    @mock.patch('yaml.load')
+    def test_partition_scheme(self, mock_lbd, mock_image_meta):
-    @mock.patch.object(utils, 'init_http_request')
-    @mock.patch.object(hu, 'list_block_devices')
-    def test_partition_scheme(self, mock_lbd, mock_http_req, mock_yaml):
         mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
         drv = nailgun.Nailgun(PROVISION_SAMPLE_DATA)
         p_scheme = drv.partition_scheme
@@ -889,10 +952,7 @@ class TestNailgun(unittest2.TestCase):
         self.assertEqual(2, len(p_scheme.vgs))
         self.assertEqual(3, len(p_scheme.parteds))
 
-    @mock.patch('yaml.load')
+    def test_image_scheme(self, mock_lbd, mock_image_meta):
-    @mock.patch.object(utils, 'init_http_request')
-    @mock.patch.object(hu, 'list_block_devices')
-    def test_image_scheme(self, mock_lbd, mock_http_req, mock_yaml):
         mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
         drv = nailgun.Nailgun(PROVISION_SAMPLE_DATA)
         p_scheme = drv.partition_scheme
@@ -920,20 +980,16 @@ class TestNailgun(unittest2.TestCase):
             self.assertIsNone(img.size)
             self.assertIsNone(img.md5)
 
-    @mock.patch.object(utils, 'init_http_request')
+    def test_image_scheme_with_checksums(self, mock_lbd, mock_image_meta):
-    @mock.patch.object(hu, 'list_block_devices')
+        fake_image_meta = {
-    def test_image_scheme_with_checksums(self, mock_lbd, mock_http_req):
+            'images': [{'raw_md5': 'fakeroot', 'raw_size': 1,
-        fake_image_meta = {'images': [{'raw_md5': 'fakeroot', 'raw_size': 1,
+                        'container_name': 'fake_image.img.gz'}]}
-                                       'container_name': 'fake_image.img.gz'}]}
+        mock_image_meta.return_value = fake_image_meta
-        prop_mock = mock.PropertyMock(return_value=yaml.dump(fake_image_meta))
-        type(mock_http_req.return_value).text = prop_mock
         mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
         p_data = PROVISION_SAMPLE_DATA.copy()
         drv = nailgun.Nailgun(p_data)
         p_scheme = drv.partition_scheme
         i_scheme = drv.image_scheme
-        mock_http_req.assert_called_once_with(
-            'http://fake.host.org:123/imgs/fake_image.yaml')
         expected_images = []
         for fs in p_scheme.fss:
             if fs.mount not in PROVISION_SAMPLE_DATA['ks_meta']['image_data']:
@@ -958,10 +1014,7 @@ class TestNailgun(unittest2.TestCase):
                 img.size, fake_image_meta['images'][0]['raw_size'])
             self.assertEqual(img.md5, fake_image_meta['images'][0]['raw_md5'])
 
-    @mock.patch('yaml.load')
+    def test_getlabel(self, mock_lbd, mock_image_meta):
-    @mock.patch.object(utils, 'init_http_request')
-    @mock.patch.object(hu, 'list_block_devices')
-    def test_getlabel(self, mock_lbd, mock_http_req, mock_yaml):
         mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
         drv = nailgun.Nailgun(PROVISION_SAMPLE_DATA)
         self.assertEqual('', drv._getlabel(None))
@@ -969,10 +1022,7 @@ class TestNailgun(unittest2.TestCase):
         self.assertEqual(' -L %s ' % long_label[:12],
                          drv._getlabel(long_label))
 
-    @mock.patch('yaml.load')
+    def test_disk_dev_not_found(self, mock_lbd, mock_image_meta):
-    @mock.patch.object(utils, 'init_http_request')
-    @mock.patch.object(hu, 'list_block_devices')
-    def test_disk_dev_not_found(self, mock_lbd, mock_http_req, mock_yaml):
         mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
         drv = nailgun.Nailgun(PROVISION_SAMPLE_DATA)
         fake_ks_disk = {
@@ -985,19 +1035,13 @@ class TestNailgun(unittest2.TestCase):
         self.assertRaises(errors.DiskNotFoundError, drv._disk_dev,
                           fake_ks_disk)
 
-    @mock.patch('yaml.load')
+    def test_get_partition_count(self, mock_lbd, mock_image_meta):
-    @mock.patch.object(utils, 'init_http_request')
-    @mock.patch.object(hu, 'list_block_devices')
-    def test_get_partition_count(self, mock_lbd, mock_http_req, mock_yaml):
         mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
         drv = nailgun.Nailgun(PROVISION_SAMPLE_DATA)
         self.assertEqual(3, drv._get_partition_count('Boot'))
         self.assertEqual(1, drv._get_partition_count('TMP'))
 
-    @mock.patch('yaml.load')
+    def test_partition_scheme_ceph(self, mock_lbd, mock_image_meta):
-    @mock.patch.object(utils, 'init_http_request')
-    @mock.patch.object(hu, 'list_block_devices')
-    def test_partition_scheme_ceph(self, mock_lbd, mock_http_req, mock_yaml):
         # TODO(agordeev): perform better testing of ceph logic
         p_data = copy.deepcopy(PROVISION_SAMPLE_DATA)
         for i in range(0, 3):
@@ -1020,10 +1064,7 @@ class TestNailgun(unittest2.TestCase):
             self.assertEqual(CEPH_DATA['partition_guid'],
                              p_scheme.parteds[disk].partitions[part].guid)
 
-    @mock.patch('fuel_agent.drivers.nailgun.yaml.load')
+    def test_grub_centos_26(self, mock_lbd, mock_image_meta):
-    @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request')
-    @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices')
-    def test_grub_centos_26(self, mock_lbd, mock_http_req, mock_yaml):
         data = copy.deepcopy(PROVISION_SAMPLE_DATA)
         data['profile'] = 'centos'
         data['ks_meta']['kernel_lt'] = 0
@@ -1033,14 +1074,11 @@ class TestNailgun(unittest2.TestCase):
                          ' ' + data['ks_meta']['pm_data']['kernel_params'])
         self.assertEqual(drv.grub.kernel_regexp, r'^vmlinuz-2\.6.*')
         self.assertEqual(drv.grub.initrd_regexp, r'^initramfs-2\.6.*')
-        self.assertIsNone(drv.grub.version)
+        self.assertEqual(1, drv.grub.version)
         self.assertIsNone(drv.grub.kernel_name)
         self.assertIsNone(drv.grub.initrd_name)
 
-    @mock.patch('fuel_agent.drivers.nailgun.yaml.load')
+    def test_grub_centos_lt(self, mock_lbd, mock_image_meta):
-    @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request')
-    @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices')
-    def test_grub_centos_lt(self, mock_lbd, mock_http_req, mock_yaml):
         data = copy.deepcopy(PROVISION_SAMPLE_DATA)
         data['profile'] = 'centos'
         data['ks_meta']['kernel_lt'] = 1
@@ -1050,14 +1088,11 @@ class TestNailgun(unittest2.TestCase):
                          ' ' + data['ks_meta']['pm_data']['kernel_params'])
         self.assertIsNone(drv.grub.kernel_regexp)
         self.assertIsNone(drv.grub.initrd_regexp)
-        self.assertIsNone(drv.grub.version)
+        self.assertEqual(1, drv.grub.version)
         self.assertIsNone(drv.grub.kernel_name)
         self.assertIsNone(drv.grub.initrd_name)
 
-    @mock.patch('fuel_agent.drivers.nailgun.yaml.load')
+    def test_grub_ubuntu(self, mock_lbd, mock_image_meta):
-    @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request')
-    @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices')
-    def test_grub_ubuntu(self, mock_lbd, mock_http_req, mock_yaml):
         data = copy.deepcopy(PROVISION_SAMPLE_DATA)
         data['profile'] = 'ubuntu'
         data['ks_meta']['kernel_lt'] = 0
@@ -1065,17 +1100,13 @@ class TestNailgun(unittest2.TestCase):
         drv = nailgun.Nailgun(data)
         self.assertEqual(drv.grub.kernel_params,
                          ' ' + data['ks_meta']['pm_data']['kernel_params'])
-        self.assertIsNone(drv.grub.version)
+        self.assertEqual(2, drv.grub.version)
         self.assertIsNone(drv.grub.kernel_regexp)
         self.assertIsNone(drv.grub.initrd_regexp)
         self.assertIsNone(drv.grub.kernel_name)
         self.assertIsNone(drv.grub.initrd_name)
 
-    @mock.patch('fuel_agent.drivers.nailgun.yaml.load')
+    def test_boot_partition_ok_single_disk(self, mock_lbd, mock_image_meta):
-    @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request')
-    @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices')
-    def test_boot_partition_ok_single_disk(self, mock_lbd,
-                                           mock_http_req, mock_yaml):
         data = copy.deepcopy(PROVISION_SAMPLE_DATA)
         data['ks_meta']['pm_data']['ks_spaces'] = SINGLE_DISK_KS_SPACES
         mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
@@ -1084,11 +1115,7 @@ class TestNailgun(unittest2.TestCase):
             drv.partition_scheme.fs_by_mount('/boot').device,
             '/dev/sda3')
 
-    @mock.patch('fuel_agent.drivers.nailgun.yaml.load')
+    def test_elevate_keep_data_single_disk(self, mock_lbd, mock_image_meta):
-    @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request')
-    @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices')
-    def test_elevate_keep_data_single_disk(self, mock_lbd,
-                                           mock_http_req, mock_yaml):
         data = copy.deepcopy(PROVISION_SAMPLE_DATA)
         data['ks_meta']['pm_data']['ks_spaces'] = SINGLE_DISK_KS_SPACES
         mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
@@ -1115,11 +1142,8 @@ class TestNailgun(unittest2.TestCase):
             if fs.mount != '/':
                 self.assertFalse(fs.keep_data)
 
-    @mock.patch('fuel_agent.drivers.nailgun.yaml.load')
-    @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request')
-    @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices')
     def test_boot_partition_ok_many_normal_disks(self, mock_lbd,
-                                                 mock_http_req, mock_yaml):
+                                                 mock_image_meta):
         data = copy.deepcopy(PROVISION_SAMPLE_DATA)
         mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
         drv = nailgun.Nailgun(data)
@@ -1127,11 +1151,8 @@ class TestNailgun(unittest2.TestCase):
             drv.partition_scheme.fs_by_mount('/boot').device,
             '/dev/sda3')
 
-    @mock.patch('fuel_agent.drivers.nailgun.yaml.load')
-    @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request')
-    @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices')
     def test_boot_partition_ok_first_disk_huge(self, mock_lbd,
-                                               mock_http_req, mock_yaml):
+                                               mock_image_meta):
         data = copy.deepcopy(PROVISION_SAMPLE_DATA)
         data['ks_meta']['pm_data']['ks_spaces'] = FIRST_DISK_HUGE_KS_SPACES
         mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
@@ -1140,11 +1161,8 @@ class TestNailgun(unittest2.TestCase):
             drv.partition_scheme.fs_by_mount('/boot').device,
             '/dev/sdb3')
 
-    @mock.patch('fuel_agent.drivers.nailgun.yaml.load')
-    @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request')
-    @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices')
     def test_boot_partition_ok_many_huge_disks(self, mock_lbd,
-                                               mock_http_req, mock_yaml):
+                                               mock_image_meta):
         data = copy.deepcopy(PROVISION_SAMPLE_DATA)
         data['ks_meta']['pm_data']['ks_spaces'] = MANY_HUGE_DISKS_KS_SPACES
         mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
@@ -1153,22 +1171,14 @@ class TestNailgun(unittest2.TestCase):
             drv.partition_scheme.fs_by_mount('/boot').device,
             '/dev/sda3')
 
-    @mock.patch('fuel_agent.drivers.nailgun.yaml.load')
+    def test_boot_partition_no_boot(self, mock_lbd, mock_image_meta):
-    @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request')
-    @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices')
-    def test_boot_partition_no_boot(self, mock_lbd,
-                                    mock_http_req, mock_yaml):
         data = copy.deepcopy(PROVISION_SAMPLE_DATA)
         data['ks_meta']['pm_data']['ks_spaces'] = NO_BOOT_KS_SPACES
         mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
         self.assertRaises(errors.WrongPartitionSchemeError,
                           nailgun.Nailgun, data)
 
-    @mock.patch('fuel_agent.drivers.nailgun.yaml.load')
+    def test_boot_partition_no_boot_nvme(self, mock_lbd, mock_image_meta):
-    @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request')
-    @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices')
-    def test_boot_partition_no_boot_nvme(self, mock_lbd,
-                                         mock_http_req, mock_yaml):
         data = copy.deepcopy(PROVISION_SAMPLE_DATA)
         data['ks_meta']['pm_data']['ks_spaces'] = ONLY_ONE_NVME_KS_SPACES
         mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE_NVME
@@ -1177,11 +1187,7 @@ class TestNailgun(unittest2.TestCase):
                 '/boot partition has not been created for some reasons'):
             nailgun.Nailgun(data)
 
-    @mock.patch('fuel_agent.drivers.nailgun.yaml.load')
+    def test_boot_partition_is_not_on_nvme(self, mock_lbd, mock_image_meta):
-    @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request')
-    @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices')
-    def test_boot_partition_is_not_on_nvme(self, mock_lbd,
-                                           mock_http_req, mock_yaml):
         data = copy.deepcopy(PROVISION_SAMPLE_DATA)
         data['ks_meta']['pm_data']['ks_spaces'] = FIRST_DISK_NVME_KS_SPACES
         mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE_NVME
@@ -1189,3 +1195,58 @@ class TestNailgun(unittest2.TestCase):
         self.assertEqual(
             drv.partition_scheme.fs_by_mount('/boot').device,
             '/dev/sda3')
+
+    def test_md_metadata_centos(self, mock_lbd, mock_image_meta):
+        data = copy.deepcopy(PROVISION_SAMPLE_DATA)
+        data['profile'] = 'base-centos-x86_64'
+        data['ks_meta']['pm_data']['ks_spaces'] = MD_RAID_KS_SPACES
+        mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
+        drv = nailgun.Nailgun(data)
+        self.assertEqual(1, drv.grub.version)
+        self.assertEqual(1, len(drv.partition_scheme.mds))
+        self.assertEqual('0.90', drv.partition_scheme.mds[0].metadata)
+
+    def test_md_metadata_centos70(self, mock_lbd, mock_image_meta):
+        data = copy.deepcopy(PROVISION_SAMPLE_DATA)
+        data['profile'] = 'base-centos7-x86_64'
+        data['ks_meta']['pm_data']['ks_spaces'] = MD_RAID_KS_SPACES
+        mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
+        drv = nailgun.Nailgun(data)
+        self.assertEqual(2, drv.grub.version)
+        self.assertEqual(1, len(drv.partition_scheme.mds))
+        self.assertEqual('default', drv.partition_scheme.mds[0].metadata)
+
+    def test_md_metadata_ubuntu(self, mock_lbd, mock_image_meta):
+        data = copy.deepcopy(PROVISION_SAMPLE_DATA)
+        data['profile'] = 'base-ubuntu_1404_x86_64'
+        data['ks_meta']['pm_data']['ks_spaces'] = MD_RAID_KS_SPACES
+        mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
+        drv = nailgun.Nailgun(data)
+        self.assertEqual(1, len(drv.partition_scheme.mds))
+        self.assertEqual(2, drv.grub.version)
+        self.assertEqual('default', drv.partition_scheme.mds[0].metadata)
+
+
+@mock.patch.object(utils, 'init_http_request')
+@mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices')
+class TestNailgunImageMeta(unittest2.TestCase):
+    def test_parse_image_meta(self, mock_lbd, mock_http_req):
+        fake_image_meta = {'images': [{'raw_md5': 'fakeroot', 'raw_size': 1,
+                                       'container_name': 'fake_image.img.gz'}]}
+        prop_mock = mock.PropertyMock(return_value=yaml.dump(fake_image_meta))
+        type(mock_http_req.return_value).text = prop_mock
+        mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
+        p_data = PROVISION_SAMPLE_DATA.copy()
+        drv = nailgun.Nailgun(p_data)
+        self.assertEqual(fake_image_meta, drv._image_meta)
+        mock_http_req.assert_called_once_with(
+            'http://fake.host.org:123/imgs/fake_image.yaml')
+
+    def test_parse_image_meta_not_parsed(self, mock_lbd, mock_http_req):
+        mock_http_req.side_effect = KeyError()
+        mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
+        p_data = PROVISION_SAMPLE_DATA.copy()
+        drv = nailgun.Nailgun(p_data)
+        self.assertEqual({}, drv._image_meta)
+        mock_http_req.assert_called_once_with(
+            'http://fake.host.org:123/imgs/fake_image.yaml')

fuel_agent/tests/test_nailgun_build_image.py

@@ -125,7 +125,8 @@ class TestNailgunBuildImage(unittest2.TestCase):
         mock_ub_instance = mock_ub.return_value
         mock_ub_instance.packages = data['packages']
         driver = NailgunBuildImage(data)
-        mock_ub.assert_called_once_with(repos=[], packages=data['packages'])
+        mock_ub.assert_called_once_with(repos=[], packages=data['packages'],
+                                        major=14, minor=4)
         self.assertEqual(driver.operating_system.packages, data['packages'])
 
     @mock.patch('fuel_agent.objects.Ubuntu')
@@ -140,7 +141,8 @@ class TestNailgunBuildImage(unittest2.TestCase):
         mock_ub_instance.packages = NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES
         driver = NailgunBuildImage(data)
         mock_ub.assert_called_once_with(
-            repos=[], packages=NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES)
+            repos=[], packages=NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES,
+            major=14, minor=4)
         self.assertEqual(driver.operating_system.packages,
                          NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES)
 
@@ -170,7 +172,8 @@ class TestNailgunBuildImage(unittest2.TestCase):
         mock_ub_instance = mock_ub.return_value
         mock_ub_instance.repos = repos
         mock_ub.assert_called_once_with(
-            repos=repos, packages=NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES)
+            repos=repos, packages=NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES,
+            major=14, minor=4)
         self.assertEqual(mock_deb_expected_calls,
                          mock_deb.call_args_list[:len(REPOS_SAMPLE)])
         self.assertEqual(driver.operating_system.repos, repos)

fuel_agent/tests/test_partition.py

@@ -67,6 +67,7 @@ class TestMultipleDevice(unittest2.TestCase):
             'level': 'level',
             'devices': ['device_a', ],
             'spares': ['device_b', ],
+            'metadata': 'default',
             'keep_data': False,
         }
         new_md = objects.MultipleDevice.from_dict(serialized)

fuel_agent/tests/test_simple_nailgun_driver.py

@@ -24,6 +24,8 @@ from fuel_agent.tests import base
 
 @mock.patch.multiple(
     simple.NailgunSimpleDriver,
+    parse_operating_system=lambda x: objects.OperatingSystem(None, None),
+    parse_image_meta=lambda x: {},
     parse_grub=lambda x: objects.Grub(),
     parse_configdrive_scheme=lambda x: objects.ConfigDriveScheme(),
     parse_image_scheme=lambda x: objects.ImageScheme())

fuel_agent/utils/md.py

@@ -76,7 +76,7 @@ def mddisplay(names=None):
     return mds
 
 
-def mdcreate(mdname, level, devices):
+def mdcreate(mdname, level, devices, metadata='default'):
     mds = mddisplay()
 
     # check if md device already exists
@@ -107,7 +107,7 @@ def mdcreate(mdname, level, devices):
     # FIXME: mdadm will ask user to continue creating if any device appears to
     #       be a part of raid array. Superblock zeroing helps to avoid that.
     map(mdclean, devices)
-    utils.execute('mdadm', '--create', '--force', mdname, '-e0.90',
+    utils.execute('mdadm', '--create', '--force', mdname, '-e', metadata,
                   '--level=%s' % level,
                   '--raid-devices=%s' % len(devices), *devices,
                   check_exit_code=[0])