Move files from mount points to actual file systems
The thing is that sometimes we have file system images and mount point hierachies which are not aligned. Let's say, we have root file system image, while partition scheme says that two file systems should be created on the node: / and /var. In this case root image has /var directory with a set of files. Obviously, we need to move all these files from /var directory on the root file system to /var file system because /var directory will be used as mount point. In order to achieve this we mount all existent file systems into a flat set of temporary directories. We then try to find specific paths which correspond to mount points and move all files from these paths to corresponding file systems. Co-Authored-By: Alexey Stupnikov <astupnikov@mirantis.com> Conflicts: bareon/drivers/deploy/nailgun.py bareon/utils/fs.py fuel_agent/tests/test_manager.py Change-Id: I1aa7523055ac4bcf6f8a93e9740ccf652ed35cc1 Closes-Bug: #1537699
This commit is contained in:
Vladimir Kozhukalov
Alexander Gordeev
parent
50a310f5c7
commit
4e249de7e8
@ -223,21 +223,6 @@ class BootLoaderAction(base.BaseAction, mixins.MountableMixin):
|
||||
'w'):
|
||||
pass
|
||||
|
||||
# FIXME(kozhukalov): When we have just os-root fs image and don't
|
||||
# have os-var-log fs image while / and /var/log are supposed to be
|
||||
# separate file systems and os-var-log is mounted into
|
||||
# non-empty directory on the / file system, those files in /var/log
|
||||
# directory become unavailable.
|
||||
# The thing is that among those file there is /var/log/upstart
|
||||
# where upstart daemon writes its logs. We have specific upstart
|
||||
# job which is to flush open files once all file systems are
|
||||
# mounted.
|
||||
# This job needs upstart directory to be available on os-var-log
|
||||
# file system.
|
||||
# This is just a temporary fix and correct fix will be available
|
||||
# soon via updates.
|
||||
utils.execute('mkdir', '-p', chroot + '/var/log/upstart')
|
||||
|
||||
with open(chroot + '/etc/fstab', 'wt', encoding='utf-8') as f:
|
||||
for fs in self.driver.partition_scheme.fss:
|
||||
# TODO(kozhukalov): Think of improving the logic so as to
|
||||
|
||||
@ -14,6 +14,9 @@
|
||||
|
||||
|
||||
import os
|
||||
import shutil
|
||||
|
||||
import six
|
||||
|
||||
from bareon.actions import base
|
||||
from bareon import errors
|
||||
@ -39,6 +42,84 @@ class CopyImageAction(base.BaseAction):
|
||||
def execute(self):
|
||||
self.do_copyimage()
|
||||
|
||||
def move_files_to_their_places(self, remove_src=True):
|
||||
"""Move files from mount points to where those files should be.
|
||||
|
||||
:param remove_src: Remove source files after sync if True (default).
|
||||
"""
|
||||
|
||||
# NOTE(kozhukalov): The thing is that sometimes we
|
||||
# have file system images and mount point hierachies
|
||||
# which are not aligned. Let's say, we have root file system
|
||||
# image, while partition scheme says that two file systems should
|
||||
# be created on the node: / and /var.
|
||||
# In this case root image has /var directory with a set of files.
|
||||
# Obviously, we need to move all these files from /var directory
|
||||
# on the root file system to /var file system because /var
|
||||
# directory will be used as mount point.
|
||||
# In order to achieve this we mount all existent file
|
||||
# systems into a flat set of temporary directories. We then
|
||||
# try to find specific paths which correspond to mount points
|
||||
# and move all files from these paths to corresponding file systems.
|
||||
|
||||
mount_map = self.mount_target_flat()
|
||||
for fs_mount in sorted(mount_map):
|
||||
head, tail = os.path.split(fs_mount)
|
||||
while head != fs_mount:
|
||||
LOG.debug('Trying to move files for %s file system', fs_mount)
|
||||
if head in mount_map:
|
||||
LOG.debug('File system %s is mounted into %s',
|
||||
head, mount_map[head])
|
||||
check_path = os.path.join(mount_map[head], tail)
|
||||
LOG.debug('Trying to check if path %s exists', check_path)
|
||||
if os.path.exists(check_path):
|
||||
LOG.debug('Path %s exists. Trying to sync all files '
|
||||
'from there to %s', mount_map[fs_mount])
|
||||
src_path = check_path + '/'
|
||||
utils.execute('rsync', '-avH', src_path,
|
||||
mount_map[fs_mount])
|
||||
if remove_src:
|
||||
shutil.rmtree(check_path)
|
||||
break
|
||||
if head == '/':
|
||||
break
|
||||
head, _tail = os.path.split(head)
|
||||
tail = os.path.join(_tail, tail)
|
||||
self.umount_target_flat(mount_map)
|
||||
|
||||
def mount_target_flat(self):
|
||||
"""Mount a set of file systems into a set of temporary directories
|
||||
|
||||
:returns: Mount map dict
|
||||
"""
|
||||
|
||||
LOG.debug('Mounting target file systems into a flat set '
|
||||
'of temporary directories')
|
||||
mount_map = {}
|
||||
for fs in self.driver.partition_scheme.fss:
|
||||
if fs.mount == 'swap':
|
||||
continue
|
||||
# It is an ugly hack to resolve python2/3 encoding issues and
|
||||
# should be removed after transistion to python3
|
||||
try:
|
||||
type(fs.mount) is unicode
|
||||
fs_mount = fs.mount.encode('ascii', 'ignore')
|
||||
except NameError:
|
||||
fs_mount = fs.mount
|
||||
mount_map[fs_mount] = fu.mount_fs_temp(fs.type, str(fs.device))
|
||||
LOG.debug('Flat mount map: %s', mount_map)
|
||||
return mount_map
|
||||
|
||||
def umount_target_flat(self, mount_map):
|
||||
"""Umount file systems previously mounted into temporary directories.
|
||||
|
||||
:param mount_map: Mount map dict
|
||||
"""
|
||||
|
||||
for mount_point in six.itervalues(mount_map):
|
||||
fu.umount_fs(mount_point)
|
||||
shutil.rmtree(mount_point)
|
||||
|
||||
def do_copyimage(self):
|
||||
LOG.debug('--- Copying images (do_copyimage) ---')
|
||||
for image in self.driver.image_scheme.images:
|
||||
@ -98,3 +179,4 @@ class CopyImageAction(base.BaseAction):
|
||||
LOG.debug('Extending %s %s' %
|
||||
(image.format, image.target_device))
|
||||
fu.extend_fs(image.format, image.target_device)
|
||||
self.move_files_to_their_places()
|
||||
|
||||
@ -205,3 +205,74 @@ class TestCopyImageAction(unittest2.TestCase):
|
||||
self.assertEqual(2, len(self.drv.image_scheme.images))
|
||||
self.assertRaises(errors.ImageChecksumMismatchError,
|
||||
self.action.execute)
|
||||
|
||||
@mock.patch('bareon.utils.fs.mount_fs_temp')
|
||||
def test_mount_target_flat(self, mock_mfst):
|
||||
def mfst_side_effect(*args, **kwargs):
|
||||
if '/dev/fake1' in args:
|
||||
return '/tmp/dir1'
|
||||
elif '/dev/fake2' in args:
|
||||
return '/tmp/dir2'
|
||||
mock_mfst.side_effect = mfst_side_effect
|
||||
self.drv.partition_scheme = objects.PartitionScheme()
|
||||
self.drv.partition_scheme.add_fs(
|
||||
device='/dev/fake1', mount='/', fs_type='ext4')
|
||||
self.drv.partition_scheme.add_fs(
|
||||
device='/dev/fake2', mount='/var/lib', fs_type='ext4')
|
||||
self.assertEqual({'/': '/tmp/dir1', '/var/lib': '/tmp/dir2'},
|
||||
self.action.mount_target_flat())
|
||||
self.assertEqual([mock.call('ext4', '/dev/fake1'),
|
||||
mock.call('ext4', '/dev/fake2')],
|
||||
mock_mfst.call_args_list)
|
||||
|
||||
@mock.patch('bareon.actions.copyimage.shutil.rmtree')
|
||||
@mock.patch('bareon.utils.fs.umount_fs')
|
||||
def test_umount_target_flat(self, mock_umfs, mock_rmtree):
|
||||
mount_map = {'/': '/tmp/dir1', '/var/lib': '/tmp/dir2'}
|
||||
self.action.umount_target_flat(mount_map)
|
||||
mock_umfs.assert_has_calls(
|
||||
[mock.call('/tmp/dir1'), mock.call('/tmp/dir2')],
|
||||
any_order=True)
|
||||
|
||||
@mock.patch('bareon.actions.copyimage.shutil.rmtree')
|
||||
@mock.patch('bareon.actions.copyimage.os.path.exists')
|
||||
@mock.patch('bareon.actions.copyimage.utils.execute')
|
||||
@mock.patch('bareon.actions.copyimage.CopyImageAction.umount_target_flat')
|
||||
@mock.patch('bareon.actions.copyimage.CopyImageAction.mount_target_flat')
|
||||
def test_move_files_to_their_places(self, mock_mtf, mock_utf,
|
||||
mock_ute, mock_ope, mock_shrmt):
|
||||
|
||||
def ope_side_effect(path):
|
||||
if path == '/tmp/dir1/var/lib':
|
||||
return True
|
||||
|
||||
mock_ope.side_effect = ope_side_effect
|
||||
mock_mtf.return_value = {'/': '/tmp/dir1', '/var/lib': '/tmp/dir2'}
|
||||
self.action.move_files_to_their_places()
|
||||
self.assertEqual(
|
||||
[mock.call('rsync', '-avH', '/tmp/dir1/var/lib/', '/tmp/dir2')],
|
||||
mock_ute.call_args_list)
|
||||
self.assertEqual(
|
||||
[mock.call('/tmp/dir1/var/lib')],
|
||||
mock_shrmt.call_args_list)
|
||||
|
||||
@mock.patch('bareon.actions.copyimage.shutil.rmtree')
|
||||
@mock.patch('bareon.actions.copyimage.os.path.exists')
|
||||
@mock.patch('bareon.actions.copyimage.utils.execute')
|
||||
@mock.patch('bareon.actions.copyimage.CopyImageAction.umount_target_flat')
|
||||
@mock.patch('bareon.actions.copyimage.CopyImageAction.mount_target_flat')
|
||||
def test_move_files_to_their_places_not_remove(self, mock_mtf, mock_utf,
|
||||
mock_ute, mock_ope,
|
||||
mock_shrmt):
|
||||
|
||||
def ope_side_effect(path):
|
||||
if path == '/tmp/dir1/var/lib':
|
||||
return True
|
||||
|
||||
mock_ope.side_effect = ope_side_effect
|
||||
mock_mtf.return_value = {'/': '/tmp/dir1', '/var/lib': '/tmp/dir2'}
|
||||
self.action.move_files_to_their_places(remove_src=False)
|
||||
self.assertEqual(
|
||||
[mock.call('rsync', '-avH', '/tmp/dir1/var/lib/', '/tmp/dir2')],
|
||||
mock_ute.call_args_list)
|
||||
self.assertFalse(mock_shrmt.called)
|
||||
|
||||
@ -170,6 +170,14 @@ class TestFSUtils(unittest2.TestCase):
|
||||
'/dev/sda4')
|
||||
self.assertEqual(ret, 'megafs')
|
||||
|
||||
@mock.patch('bareon.utils.fs.mount_fs')
|
||||
@mock.patch('bareon.utils.fs.tempfile.mkdtemp')
|
||||
def test_mount_fs_temp(self, mock_mkdtemp, mock_mount, mock_exec):
|
||||
mock_mkdtemp.return_value = '/tmp/dir'
|
||||
self.assertEqual('/tmp/dir', fu.mount_fs_temp('ext4', '/dev/fake'))
|
||||
mock_mkdtemp.assert_called_once_with(dir=None, suffix='')
|
||||
mock_mount.assert_called_once_with('ext4', '/dev/fake', '/tmp/dir')
|
||||
|
||||
|
||||
class TestFSRetry(unittest2.TestCase):
|
||||
|
||||
|
||||
@ -13,6 +13,7 @@
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import uuid
|
||||
|
||||
from bareon import errors
|
||||
@ -134,3 +135,9 @@ def get_fs_type(device):
|
||||
output = utils.execute('blkid', '-o', 'value', '-s', 'TYPE',
|
||||
'-c', '/dev/null', device)[0]
|
||||
return output.strip()
|
||||
|
||||
|
||||
def mount_fs_temp(fs_type, fs_dev, tmpdir=None, suffix=''):
|
||||
mount_point = tempfile.mkdtemp(dir=tmpdir, suffix=suffix)
|
||||
mount_fs(fs_type, fs_dev, mount_point)
|
||||
return mount_point
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user