From 9c5e3579bf95c10e3b72cb77ae98fba9b758b193 Mon Sep 17 00:00:00 2001 From: Vladimir Kozhukalov Date: Tue, 31 Mar 2015 19:25:20 +0300 Subject: [PATCH] IBP: Added driver and objects for building images As far as building of OS images is nothing more than just a stage of the whole OS installing procedure it is sounds rational to implement this in terms of fuel-agent. Besides, we already have plenty of utilities which could be useful during building of images. And some tasks are the same like pre-configuring some files inside target OS. Related-bug: #1433193 Implements: blueprint ibp-build-ubuntu-images Change-Id: I3fadfb16e06e4ee16926da29b7b83ca005500698 --- etc/fuel-agent/fuel-agent.conf.sample | 66 ++++- fuel_agent/cmd/agent.py | 35 ++- fuel_agent/drivers/base.py | 27 ++ fuel_agent/drivers/nailgun.py | 134 ++++++++- fuel_agent/errors.py | 12 + fuel_agent/manager.py | 287 +++++++++++++++++-- fuel_agent/objects/__init__.py | 8 + fuel_agent/objects/device.py | 28 ++ fuel_agent/objects/operating_system.py | 23 ++ fuel_agent/objects/partition.py | 11 + fuel_agent/objects/repo.py | 28 ++ fuel_agent/tests/test_manager.py | 182 ++++++++++++ fuel_agent/tests/test_nailgun.py | 8 +- fuel_agent/tests/test_nailgun_build_image.py | 244 ++++++++++++++++ fuel_agent/tests/test_utils.py | 28 +- fuel_agent/utils/utils.py | 14 +- requirements.txt | 2 +- setup.cfg | 11 +- 18 files changed, 1080 insertions(+), 68 deletions(-) create mode 100644 fuel_agent/drivers/base.py create mode 100644 fuel_agent/objects/device.py create mode 100644 fuel_agent/objects/operating_system.py create mode 100644 fuel_agent/objects/repo.py create mode 100644 fuel_agent/tests/test_nailgun_build_image.py diff --git a/etc/fuel-agent/fuel-agent.conf.sample b/etc/fuel-agent/fuel-agent.conf.sample index 51e7f53..0359d21 100644 --- a/etc/fuel-agent/fuel-agent.conf.sample +++ b/etc/fuel-agent/fuel-agent.conf.sample @@ -17,13 +17,34 @@ # value) #config_drive_path=/tmp/config-drive.img +# Path where to store actual rules for udev daemon (string +# value) +#udev_rules_dir=/etc/udev/rules.d + +# Path where to store default rules for udev daemon (string +# value) +#udev_rules_lib_dir=/lib/udev/rules.d + +# Substring to which file extension .rules be renamed (string +# value) +#udev_rename_substr=.renamedrule + +# Directory where we build images (string value) +#image_build_dir=/tmp + +# Directory where we build images (string value) +#image_build_suffix=.fuel-agent-image + # # Options defined in fuel_agent.cmd.agent # -# Provision data file (string value) -#provision_data_file=/tmp/provision.json +# Input data file (string value) +#input_data_file=/tmp/provision.json + +# Input data (json string) (string value) +#input_data= # @@ -99,6 +120,7 @@ logging_debug_format_suffix= # Deprecated group/name - [DEFAULT]/logfile log_file=/var/log/fuel-agent.log + # (Optional) The base directory used for relative --log-file # paths. (string value) # Deprecated group/name - [DEFAULT]/logdir @@ -120,3 +142,43 @@ log_file=/var/log/fuel-agent.log #syslog_log_facility=LOG_USER +# +# Options defined in fuel_agent.utils.artifact_utils +# + +# Size of data chunk to operate with images (integer value) +#data_chunk_size=1048576 + + +# +# Options defined in fuel_agent.utils.build_utils +# + +# Maximum allowed loop devices count to use (integer value) +#max_loop_count=255 + +# Size of sparse file in MiBs (integer value) +#sparse_file_size=2048 + +# System-wide major number for loop device (integer value) +#loop_dev_major=7 + + +# +# Options defined in fuel_agent.utils.utils +# + +# Maximum retries count for http requests. 0 means infinite +# (integer value) +#http_max_retries=30 + +# Http request timeout in seconds (floating point value) +#http_request_timeout=10.0 + +# Delay in seconds before the next http request retry +# (floating point value) +#http_retry_delay=2.0 + +# Block size of data to read for calculating checksum (integer +# value) +#read_chunk_size=1048576 diff --git a/fuel_agent/cmd/agent.py b/fuel_agent/cmd/agent.py index 90412ae..138bbb7 100644 --- a/fuel_agent/cmd/agent.py +++ b/fuel_agent/cmd/agent.py @@ -12,30 +12,36 @@ # License for the specific language governing permissions and limitations # under the License. -import json import sys from oslo.config import cfg +from oslo_serialization import jsonutils as json import six from fuel_agent import manager as manager -from fuel_agent.openstack.common import log +from fuel_agent.openstack.common import log as logging from fuel_agent import version -opts = [ +cli_opts = [ cfg.StrOpt( - 'provision_data_file', + 'input_data_file', default='/tmp/provision.json', - help='Provision data file' + help='Input data file' + ), + cfg.StrOpt( + 'input_data', + default='', + help='Input data (json string)' ), ] CONF = cfg.CONF -CONF.register_opts(opts) +CONF.register_cli_opts(cli_opts) CONF(sys.argv[1:], project='fuel-agent', version=version.version_info.release_string()) -log.setup('fuel-agent') -LOG = log.getLogger(__name__) + +logging.setup('fuel-agent') +LOG = logging.getLogger(__name__) def provision(): @@ -58,6 +64,10 @@ def bootloader(): main(['do_bootloader']) +def build_image(): + main(['do_build_image']) + + def print_err(line): sys.stderr.write(six.text_type(line)) sys.stderr.write('\n') @@ -72,8 +82,12 @@ def handle_exception(exc): def main(actions=None): try: - with open(CONF.provision_data_file) as f: - data = json.load(f) + if CONF.input_data: + data = json.loads(CONF.input_data) + else: + with open(CONF.input_data_file) as f: + data = json.load(f) + LOG.debug('Input data: %s', data) mgr = manager.Manager(data) if actions: @@ -82,5 +96,6 @@ def main(actions=None): except Exception as exc: handle_exception(exc) + if __name__ == '__main__': main() diff --git a/fuel_agent/drivers/base.py b/fuel_agent/drivers/base.py new file mode 100644 index 0000000..819132d --- /dev/null +++ b/fuel_agent/drivers/base.py @@ -0,0 +1,27 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +import six + + +@six.add_metaclass(abc.ABCMeta) +class BaseDataDriver(object): + """Data driver API is to be put here. + For example, data validation methods, + methods for getting object schemes, etc. + """ + + def __init__(self, data): + self.data = data diff --git a/fuel_agent/drivers/nailgun.py b/fuel_agent/drivers/nailgun.py index 1c225de..c877ca1 100644 --- a/fuel_agent/drivers/nailgun.py +++ b/fuel_agent/drivers/nailgun.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import itertools import math import os import six +import yaml +from six.moves.urllib.parse import urljoin +from six.moves.urllib.parse import urlparse +from six.moves.urllib.parse import urlsplit + +from fuel_agent.drivers.base import BaseDataDriver from fuel_agent.drivers import ks_spaces_validator from fuel_agent import errors from fuel_agent import objects @@ -23,9 +30,6 @@ from fuel_agent.openstack.common import log as logging from fuel_agent.utils import hardware_utils as hu from fuel_agent.utils import utils -from six.moves.urllib.parse import urljoin -from six.moves.urllib.parse import urlparse -import yaml LOG = logging.getLogger(__name__) @@ -60,11 +64,9 @@ def match_device(hu_disk, ks_disk): return False -class Nailgun(object): +class Nailgun(BaseDataDriver): def __init__(self, data): - # Here data is expected to be raw provisioning data - # how it is given by nailgun - self.data = data + super(Nailgun, self).__init__(data) # this var is used as a flag that /boot fs # has already been added. we need this to @@ -394,16 +396,13 @@ class Nailgun(object): filename = os.path.basename(urlparse(root_uri).path).split('.')[0] + \ '.yaml' metadata_url = urljoin(root_uri, filename) - image_meta = {} - raw_image_meta = None try: - raw_image_meta = yaml.load( + image_meta = yaml.load( utils.init_http_request(metadata_url).text) except Exception as e: LOG.exception(e) LOG.debug('Failed to fetch/decode image meta data') - if raw_image_meta: - [image_meta.update(img_info) for img_info in raw_image_meta] + image_meta = {} # We assume for every file system user may provide a separate # file system image. For example if partitioning scheme has # /, /boot, /var/lib file systems then we will try to get images @@ -415,13 +414,120 @@ class Nailgun(object): LOG.debug('Adding image for fs %s: uri=%s format=%s container=%s' % (mount_point, image_data['uri'], image_data['format'], image_data['container'])) + iname = os.path.basename(urlparse(image_data['uri']).path) + imeta = next(itertools.chain( + (img for img in image_meta.get('images', []) + if img['container_name'] == iname), [{}])) image_scheme.add_image( uri=image_data['uri'], target_device=self.partition_scheme.fs_by_mount( mount_point).device, format=image_data['format'], container=image_data['container'], - size=image_meta.get(mount_point, {}).get('size'), - md5=image_meta.get(mount_point, {}).get('md5'), + size=imeta.get('raw_size'), + md5=imeta.get('raw_md5'), ) return image_scheme + + +class NailgunBuildImage(BaseDataDriver): + + # TODO(kozhukalov): + # This list of packages is used by default only if another + # list isn't given in build image data. In the future + # we need to handle package list in nailgun. Even more, + # in the future, we'll be building not only ubuntu images + # and we'll likely move this list into some kind of config. + DEFAULT_TRUSTY_PACKAGES = [ + "acl", + "anacron", + "bash-completion", + "bridge-utils", + "bsdmainutils", + "build-essential", + "cloud-init", + "curl", + "daemonize", + "debconf-utils", + "gdisk", + "grub-pc", + "linux-firmware", + "linux-firmware-nonfree", + "linux-headers-generic-lts-trusty", + "linux-image-generic-lts-trusty", + "lvm2", + "mcollective", + "mdadm", + "nailgun-agent", + "nailgun-mcagents", + "nailgun-net-check", + "ntp", + "openssh-client", + "openssh-server", + "puppet", + "python-amqp", + "ruby-augeas", + "ruby-ipaddress", + "ruby-json", + "ruby-netaddr", + "ruby-openstack", + "ruby-shadow", + "ruby-stomp", + "telnet", + "ubuntu-minimal", + "ubuntu-standard", + "uuid-runtime", + "vim", + "virt-what", + "vlan", + ] + + def __init__(self, data): + super(NailgunBuildImage, self).__init__(data) + self.parse_schemes() + self.parse_operating_system() + + def parse_operating_system(self): + if self.data.get('codename').lower() != 'trusty': + raise errors.WrongInputDataError( + 'Currently, only Ubuntu Trusty is supported, given ' + 'codename is {0}'.format(self.data.get('codename'))) + + packages = self.data.get('packages', self.DEFAULT_TRUSTY_PACKAGES) + + repos = [] + for repo in self.data['repos']: + repos.append(objects.DEBRepo( + name=repo['name'], + uri=repo['uri'], + suite=repo['suite'], + section=repo['section'], + priority=repo['priority'])) + + self.operating_system = objects.Ubuntu(repos=repos, packages=packages) + + def parse_schemes(self): + self.image_scheme = objects.ImageScheme() + self.partition_scheme = objects.PartitionScheme() + + for mount, image in six.iteritems(self.data['image_data']): + filename = os.path.basename(urlsplit(image['uri']).path) + # Loop does not allocate any loop device + # during initialization. + device = objects.Loop() + + self.image_scheme.add_image( + uri='file://' + os.path.join(self.data['output'], filename), + format=image['format'], + container=image['container'], + target_device=device) + + self.partition_scheme.add_fs( + device=device, + mount=mount, + fs_type=image['format']) + + if mount == '/': + metadata_filename = filename.split('.', 1)[0] + '.yaml' + self.metadata_uri = 'file://' + os.path.join( + self.data['output'], metadata_filename) diff --git a/fuel_agent/errors.py b/fuel_agent/errors.py index 6c686e4..f960a9b 100644 --- a/fuel_agent/errors.py +++ b/fuel_agent/errors.py @@ -19,6 +19,10 @@ class BaseError(Exception): super(BaseError, self).__init__(message, *args, **kwargs) +class WrongInputDataError(BaseError): + pass + + class WrongPartitionSchemeError(BaseError): pass @@ -146,3 +150,11 @@ class ImageChecksumMismatchError(BaseError): class NoFreeLoopDevices(BaseError): pass + + +class WrongRepositoryError(BaseError): + pass + + +class WrongDeviceError(BaseError): + pass diff --git a/fuel_agent/manager.py b/fuel_agent/manager.py index 1abc765..c0575c9 100644 --- a/fuel_agent/manager.py +++ b/fuel_agent/manager.py @@ -13,12 +13,18 @@ # limitations under the License. import os +import shutil +import signal +import tempfile +import time +import yaml from oslo.config import cfg from fuel_agent import errors from fuel_agent.openstack.common import log as logging from fuel_agent.utils import artifact_utils as au +from fuel_agent.utils import build_utils as bu from fuel_agent.utils import fs_utils as fu from fuel_agent.utils import grub_utils as gu from fuel_agent.utils import lvm_utils as lu @@ -27,11 +33,6 @@ from fuel_agent.utils import partition_utils as pu from fuel_agent.utils import utils opts = [ - cfg.StrOpt( - 'data_driver', - default='nailgun', - help='Data driver' - ), cfg.StrOpt( 'nc_template_path', default='/usr/share/fuel-agent/cloud-init-templates', @@ -67,10 +68,29 @@ opts = [ default='empty_rule', help='Correct empty rule for udev daemon', ), + cfg.StrOpt( + 'image_build_dir', + default='/tmp', + help='Directory where the image is supposed to be built', + ), + cfg.StrOpt( + 'image_build_suffix', + default='.fuel-agent-image', + help='Suffix which is used while creating temporary files', + ), +] + +cli_opts = [ + cfg.StrOpt( + 'data_driver', + default='nailgun', + help='Data driver' + ), ] CONF = cfg.CONF CONF.register_opts(opts) +CONF.register_cli_opts(cli_opts) LOG = logging.getLogger(__name__) @@ -311,41 +331,49 @@ class Manager(object): (image.format, image.target_device)) fu.extend_fs(image.format, image.target_device) - def mount_target(self, chroot): + # TODO(kozhukalov): write tests + def mount_target(self, chroot, treat_mtab=True, pseudo=True): + """Mount a set of file systems into a chroot + + :param chroot: Directory where to mount file systems + :param treat_mtab: If mtab needs to be actualized (Default: True) + :param pseudo: If pseudo file systems + need to be mounted (Default: True) + """ LOG.debug('Mounting target file systems') # Here we are going to mount all file systems in partition scheme. - # Shorter paths earlier. We sort all mount points by their depth. - # ['/', '/boot', '/var', '/var/lib/mysql'] - key = lambda x: len(x.mount.rstrip('/').split('/')) - for fs in sorted(self.driver.partition_scheme.fss, key=key): + for fs in self.driver.partition_scheme.fs_sorted_by_depth(): if fs.mount == 'swap': continue mount = chroot + fs.mount - if not os.path.isdir(mount): - os.makedirs(mount, mode=0o755) - fu.mount_fs(fs.type, fs.device, mount) - fu.mount_bind(chroot, '/sys') - fu.mount_bind(chroot, '/dev') - fu.mount_bind(chroot, '/proc') - mtab = utils.execute( - 'chroot', chroot, 'grep', '-v', 'rootfs', '/proc/mounts')[0] - mtab_path = chroot + '/etc/mtab' - if os.path.islink(mtab_path): - os.remove(mtab_path) - with open(mtab_path, 'wb') as f: - f.write(mtab) + utils.makedirs_if_not_exists(mount) + fu.mount_fs(fs.type, str(fs.device), mount) - def umount_target(self, chroot): + if pseudo: + for path in ('/sys', '/dev', '/proc'): + utils.makedirs_if_not_exists(chroot + path) + fu.mount_bind(chroot, path) + + if treat_mtab: + mtab = utils.execute( + 'chroot', chroot, 'grep', '-v', 'rootfs', '/proc/mounts')[0] + mtab_path = chroot + '/etc/mtab' + if os.path.islink(mtab_path): + os.remove(mtab_path) + with open(mtab_path, 'wb') as f: + f.write(mtab) + + # TODO(kozhukalov): write tests + def umount_target(self, chroot, pseudo=True): LOG.debug('Umounting target file systems') - fu.umount_fs(chroot + '/proc') - fu.umount_fs(chroot + '/dev') - fu.umount_fs(chroot + '/sys') - key = lambda x: len(x.mount.rstrip('/').split('/')) - for fs in sorted(self.driver.partition_scheme.fss, - key=key, reverse=True): + if pseudo: + for path in ('/proc', '/dev', '/sys'): + fu.umount_fs(chroot + path) + for fs in self.driver.partition_scheme.fs_sorted_by_depth( + reverse=True): if fs.mount == 'swap': continue - fu.umount_fs(fs.device) + fu.umount_fs(chroot + fs.mount) def do_bootloader(self): LOG.debug('--- Installing bootloader (do_bootloader) ---') @@ -416,3 +444,200 @@ class Manager(object): self.do_configdrive() self.do_copyimage() self.do_bootloader() + LOG.debug('--- Provisioning END (do_provisioning) ---') + + # TODO(kozhukalov): Split this huge method + # into a set of smaller ones + # https://bugs.launchpad.net/fuel/+bug/1444090 + def do_build_image(self): + """Building OS images includes the following steps + 1) create temporary sparse files for all images (truncate) + 2) attach temporary files to loop devices (losetup) + 3) create file systems on these loop devices + 4) create temporary chroot directory + 5) mount loop devices into chroot directory + 6) install operating system (debootstrap and apt-get) + 7) configure OS (clean sources.list and preferences, etc.) + 8) umount loop devices + 9) resize file systems on loop devices + 10) shrink temporary sparse files (images) + 11) containerize (gzip) temporary sparse files + 12) move temporary gzipped files to their final location + """ + LOG.info('--- Building image (do_build_image) ---') + # TODO(kozhukalov): Implement metadata + # as a pluggable data driver to avoid any fixed format. + metadata = {} + + # TODO(kozhukalov): implement this using image metadata + # we need to compare list of packages and repos + LOG.info('*** Checking if image exists ***') + if all([os.path.exists(img.uri.split('file://', 1)[1]) + for img in self.driver.image_scheme.images]): + LOG.debug('All necessary images are available. ' + 'Nothing needs to be done.') + return + LOG.debug('At least one of the necessary images is unavailable. ' + 'Starting build process.') + + LOG.info('*** Preparing image space ***') + for image in self.driver.image_scheme.images: + LOG.debug('Creating temporary sparsed file for the ' + 'image: %s', image.uri) + img_tmp_file = bu.create_sparse_tmp_file( + dir=CONF.image_build_dir, suffix=CONF.image_build_suffix) + LOG.debug('Temporary file: %s', img_tmp_file) + + # we need to remember those files + # to be able to shrink them and move in the end + image.img_tmp_file = img_tmp_file + + LOG.debug('Looking for a free loop device') + image.target_device.name = bu.get_free_loop_device() + + LOG.debug('Attaching temporary image file to free loop device') + bu.attach_file_to_loop(img_tmp_file, str(image.target_device)) + + # find fs with the same loop device object + # as image.target_device + fs = self.driver.partition_scheme.fs_by_device(image.target_device) + + LOG.debug('Creating file system on the image') + fu.make_fs( + fs_type=fs.type, + fs_options=fs.options, + fs_label=fs.label, + dev=str(fs.device)) + + LOG.debug('Creating temporary chroot directory') + chroot = tempfile.mkdtemp( + dir=CONF.image_build_dir, suffix=CONF.image_build_suffix) + LOG.debug('Temporary chroot: %s', chroot) + + # mounting all images into chroot tree + self.mount_target(chroot, treat_mtab=False, pseudo=False) + + LOG.info('*** Shipping image content ***') + LOG.debug('Installing operating system into image') + # FIXME(kozhukalov): !!! we need this part to be OS agnostic + + # DEBOOTSTRAP + # we use first repo as the main mirror + uri = self.driver.operating_system.repos[0].uri + suite = self.driver.operating_system.repos[0].suite + + LOG.debug('Preventing services from being get started') + bu.suppress_services_start(chroot) + LOG.debug('Installing base operating system using debootstrap') + bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot) + + # APT-GET + LOG.debug('Configuring apt inside chroot') + LOG.debug('Setting environment variables') + bu.set_apt_get_env() + LOG.debug('Allowing unauthenticated repos') + bu.pre_apt_get(chroot) + + for repo in self.driver.operating_system.repos: + LOG.debug('Adding repository source: name={name}, uri={uri},' + 'suite={suite}, section={section}'.format( + name=repo.name, uri=repo.uri, + suite=repo.suite, section=repo.section)) + bu.add_apt_source( + name=repo.name, + uri=repo.uri, + suite=repo.suite, + section=repo.section, + chroot=chroot) + LOG.debug('Adding repository preference: ' + 'name={name}, priority={priority}'.format( + name=repo.name, priority=repo.priority)) + bu.add_apt_preference( + name=repo.name, + priority=repo.priority, + suite=repo.suite, + section=repo.section, + chroot=chroot) + + metadata.setdefault('repos', []).append({ + 'type': 'deb', + 'name': repo.name, + 'uri': repo.uri, + 'suite': repo.suite, + 'section': repo.section, + 'priority': repo.priority, + 'meta': repo.meta}) + + LOG.debug('Preventing services from being get started') + bu.suppress_services_start(chroot) + + packages = self.driver.operating_system.packages + metadata['packages'] = packages + + # we need /proc to be mounted for apt-get success + proc_path = os.path.join(chroot, 'proc') + utils.makedirs_if_not_exists(proc_path) + fu.mount_bind(chroot, '/proc') + + LOG.debug('Installing packages using apt-get: %s', + ' '.join(packages)) + bu.run_apt_get(chroot, packages=packages) + + LOG.debug('Post-install OS configuration') + bu.do_post_inst(chroot) + + LOG.debug('Making sure there are no running processes ' + 'inside chroot before trying to umount chroot') + bu.send_signal_to_chrooted_processes(chroot, signal.SIGTERM) + # We assume there might be some processes which + # require some reasonable time to stop before we try + # to send them SIGKILL. Waiting for 2 seconds + # looks reasonable here. + time.sleep(2) + bu.send_signal_to_chrooted_processes(chroot, signal.SIGKILL) + + LOG.info('*** Finalizing image space ***') + fu.umount_fs(proc_path) + # umounting all loop devices + self.umount_target(chroot, pseudo=False) + + for image in self.driver.image_scheme.images: + LOG.debug('Deattaching loop device from file: %s', + image.img_tmp_file) + bu.deattach_loop(str(image.target_device)) + LOG.debug('Shrinking temporary image file: %s', + image.img_tmp_file) + bu.shrink_sparse_file(image.img_tmp_file) + + raw_size = os.path.getsize(image.img_tmp_file) + raw_md5 = utils.calculate_md5(image.img_tmp_file, raw_size) + + LOG.debug('Containerizing temporary image file: %s', + image.img_tmp_file) + img_tmp_containerized = bu.containerize( + image.img_tmp_file, image.container) + img_containerized = image.uri.split('file://', 1)[1] + + # NOTE(kozhukalov): implement abstract publisher + LOG.debug('Moving image file to the final location: %s', + img_containerized) + shutil.move(img_tmp_containerized, img_containerized) + + container_size = os.path.getsize(img_containerized) + container_md5 = utils.calculate_md5( + img_containerized, container_size) + metadata.setdefault('images', []).append({ + 'raw_md5': raw_md5, + 'raw_size': raw_size, + 'raw_name': None, + 'container_name': os.path.basename(img_containerized), + 'container_md5': container_md5, + 'container_size': container_size, + 'container': image.container, + 'format': image.format}) + + # NOTE(kozhukalov): implement abstract publisher + LOG.debug('Image metadata: %s', metadata) + with open(self.driver.metadata_uri.split('file://', 1)[1], 'w') as f: + yaml.safe_dump(metadata, stream=f) + LOG.info('--- Building image END (do_build_image) ---') diff --git a/fuel_agent/objects/__init__.py b/fuel_agent/objects/__init__.py index 33d41dd..a701b01 100644 --- a/fuel_agent/objects/__init__.py +++ b/fuel_agent/objects/__init__.py @@ -16,8 +16,11 @@ from fuel_agent.objects.configdrive import ConfigDriveCommon from fuel_agent.objects.configdrive import ConfigDriveMcollective from fuel_agent.objects.configdrive import ConfigDrivePuppet from fuel_agent.objects.configdrive import ConfigDriveScheme +from fuel_agent.objects.device import Loop from fuel_agent.objects.image import Image from fuel_agent.objects.image import ImageScheme +from fuel_agent.objects.operating_system import OperatingSystem +from fuel_agent.objects.operating_system import Ubuntu from fuel_agent.objects.partition import Fs from fuel_agent.objects.partition import Lv from fuel_agent.objects.partition import Md @@ -25,9 +28,14 @@ from fuel_agent.objects.partition import Partition from fuel_agent.objects.partition import PartitionScheme from fuel_agent.objects.partition import Pv from fuel_agent.objects.partition import Vg +from fuel_agent.objects.repo import DEBRepo +from fuel_agent.objects.repo import Repo __all__ = [ 'Partition', 'Pv', 'Vg', 'Lv', 'Md', 'Fs', 'PartitionScheme', 'ConfigDriveCommon', 'ConfigDrivePuppet', 'ConfigDriveMcollective', 'ConfigDriveScheme', 'Image', 'ImageScheme', + 'OperatingSystem', 'Ubuntu', + 'Repo', 'DEBRepo', + 'Loop', ] diff --git a/fuel_agent/objects/device.py b/fuel_agent/objects/device.py new file mode 100644 index 0000000..eec4bdd --- /dev/null +++ b/fuel_agent/objects/device.py @@ -0,0 +1,28 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from fuel_agent import errors + + +class Loop(object): + def __init__(self, name=None): + self.name = name + + def __str__(self): + if self.name: + return self.name + raise errors.WrongDeviceError( + 'Loop device can not be stringified. ' + 'Name attribute is not set. Current: ' + 'name={0}'.format(self.name)) diff --git a/fuel_agent/objects/operating_system.py b/fuel_agent/objects/operating_system.py new file mode 100644 index 0000000..3eac4ab --- /dev/null +++ b/fuel_agent/objects/operating_system.py @@ -0,0 +1,23 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class OperatingSystem(object): + def __init__(self, repos, packages): + self.repos = repos + self.packages = packages + + +class Ubuntu(OperatingSystem): + pass diff --git a/fuel_agent/objects/partition.py b/fuel_agent/objects/partition.py index c113236..00911f2 100644 --- a/fuel_agent/objects/partition.py +++ b/fuel_agent/objects/partition.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + from fuel_agent import errors from fuel_agent.openstack.common import log as logging @@ -289,6 +291,15 @@ class PartitionScheme(object): if found: return found[0] + def fs_sorted_by_depth(self, reverse=False): + """Getting file systems sorted by path length. Shorter paths earlier. + ['/', '/boot', '/var', '/var/lib/mysql'] + :param reverse: Sort backward (Default: False) + """ + def key(x): + return x.mount.rstrip(os.path.sep).count(os.path.sep) + return sorted(self.fss, key=key, reverse=reverse) + def lv_by_device_name(self, device_name): found = filter(lambda x: x.device_name == device_name, self.lvs) if found: diff --git a/fuel_agent/objects/repo.py b/fuel_agent/objects/repo.py new file mode 100644 index 0000000..f678707 --- /dev/null +++ b/fuel_agent/objects/repo.py @@ -0,0 +1,28 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class Repo(object): + def __init__(self, name, uri, priority=None): + self.name = name + self.uri = uri + self.priority = priority + + +class DEBRepo(Repo): + def __init__(self, name, uri, suite, section, meta=None, priority=None): + super(DEBRepo, self).__init__(name, uri, priority) + self.suite = suite + self.section = section + self.meta = meta diff --git a/fuel_agent/tests/test_manager.py b/fuel_agent/tests/test_manager.py index 72d3b50..8dff134 100644 --- a/fuel_agent/tests/test_manager.py +++ b/fuel_agent/tests/test_manager.py @@ -14,12 +14,14 @@ import mock import os +import signal from oslo.config import cfg from oslotest import base as test_base from fuel_agent import errors from fuel_agent import manager +from fuel_agent import objects from fuel_agent.objects import partition from fuel_agent.tests import test_nailgun from fuel_agent.utils import artifact_utils as au @@ -328,3 +330,183 @@ class TestManager(test_base.BaseTestCase): self.assertEqual(2, len(self.mgr.driver.image_scheme.images)) self.assertRaises(errors.ImageChecksumMismatchError, self.mgr.do_copyimage) + + @mock.patch('fuel_agent.manager.bu', create=True) + @mock.patch('fuel_agent.manager.fu', create=True) + @mock.patch('fuel_agent.manager.utils', create=True) + @mock.patch('fuel_agent.manager.os', create=True) + @mock.patch('fuel_agent.manager.shutil.move') + @mock.patch('fuel_agent.manager.open', + create=True, new_callable=mock.mock_open) + @mock.patch('fuel_agent.manager.tempfile.mkdtemp') + @mock.patch('fuel_agent.manager.time.sleep') + @mock.patch('fuel_agent.manager.yaml.safe_dump') + @mock.patch.object(manager.Manager, 'mount_target') + @mock.patch.object(manager.Manager, 'umount_target') + def test_do_build_image(self, mock_umount_target, mock_mount_target, + mock_yaml_dump, mock_sleep, mock_mkdtemp, + mock_open, mock_shutil_move, mock_os, + mock_utils, mock_fu, mock_bu): + + loops = [objects.Loop(), objects.Loop()] + + self.mgr.driver.image_scheme = objects.ImageScheme([ + objects.Image('file:///fake/img.img.gz', loops[0], 'ext4', 'gzip'), + objects.Image('file:///fake/img-boot.img.gz', + loops[1], 'ext2', 'gzip')]) + self.mgr.driver.partition_scheme = objects.PartitionScheme() + self.mgr.driver.partition_scheme.add_fs( + device=loops[0], mount='/', fs_type='ext4') + self.mgr.driver.partition_scheme.add_fs( + device=loops[1], mount='/boot', fs_type='ext2') + self.mgr.driver.metadata_uri = 'file:///fake/img.yaml' + self.mgr.driver.operating_system = objects.Ubuntu( + repos=[ + objects.DEBRepo('ubuntu', 'http://fakeubuntu', + 'trusty', 'fakesection'), + objects.DEBRepo('mos', 'http://fakemos', + 'mosX.Y', 'fakesection', priority=1000)], + packages=['fakepackage1', 'fakepackage2']) + + mock_os.path.exists.return_value = False + mock_os.path.join.return_value = '/tmp/imgdir/proc' + mock_os.path.basename.side_effect = ['img.img.gz', 'img-boot.img.gz'] + mock_bu.create_sparse_tmp_file.side_effect = \ + ['/tmp/img', '/tmp/img-boot'] + mock_bu.get_free_loop_device.side_effect = ['/dev/loop0', '/dev/loop1'] + mock_mkdtemp.return_value = '/tmp/imgdir' + getsize_side = [20, 2, 10, 1] + mock_os.path.getsize.side_effect = getsize_side + md5_side = ['fakemd5_raw', 'fakemd5_gzip', + 'fakemd5_raw_boot', 'fakemd5_gzip_boot'] + mock_utils.calculate_md5.side_effect = md5_side + mock_bu.containerize.side_effect = ['/tmp/img.gz', '/tmp/img-boot.gz'] + + self.mgr.do_build_image() + self.assertEqual( + [mock.call('/fake/img.img.gz'), + mock.call('/fake/img-boot.img.gz')], + mock_os.path.exists.call_args_list) + self.assertEqual([mock.call(dir=CONF.image_build_dir, + suffix=CONF.image_build_suffix)] * 2, + mock_bu.create_sparse_tmp_file.call_args_list) + self.assertEqual([mock.call()] * 2, + mock_bu.get_free_loop_device.call_args_list) + self.assertEqual([mock.call('/tmp/img', '/dev/loop0'), + mock.call('/tmp/img-boot', '/dev/loop1')], + mock_bu.attach_file_to_loop.call_args_list) + self.assertEqual([mock.call(fs_type='ext4', fs_options='', + fs_label='', dev='/dev/loop0'), + mock.call(fs_type='ext2', fs_options='', + fs_label='', dev='/dev/loop1')], + mock_fu.make_fs.call_args_list) + mock_mkdtemp.assert_called_once_with(dir=CONF.image_build_dir, + suffix=CONF.image_build_suffix) + mock_mount_target.assert_called_once_with( + '/tmp/imgdir', treat_mtab=False, pseudo=False) + self.assertEqual([mock.call('/tmp/imgdir')] * 2, + mock_bu.suppress_services_start.call_args_list) + mock_bu.run_debootstrap.assert_called_once_with( + uri='http://fakeubuntu', suite='trusty', chroot='/tmp/imgdir') + mock_bu.set_apt_get_env.assert_called_once_with() + mock_bu.pre_apt_get.assert_called_once_with('/tmp/imgdir') + self.assertEqual([ + mock.call(name='ubuntu', + uri='http://fakeubuntu', + suite='trusty', + section='fakesection', + chroot='/tmp/imgdir'), + mock.call(name='mos', + uri='http://fakemos', + suite='mosX.Y', + section='fakesection', + chroot='/tmp/imgdir')], + mock_bu.add_apt_source.call_args_list) + self.assertEqual([ + mock.call(name='ubuntu', + priority=None, + suite='trusty', + section='fakesection', + chroot='/tmp/imgdir'), + mock.call(name='mos', + priority=1000, + suite='mosX.Y', + section='fakesection', + chroot='/tmp/imgdir')], + mock_bu.add_apt_preference.call_args_list) + mock_utils.makedirs_if_not_exists.assert_called_once_with( + '/tmp/imgdir/proc') + mock_fu.mount_bind.assert_called_once_with('/tmp/imgdir', '/proc') + mock_bu.run_apt_get.assert_called_once_with( + '/tmp/imgdir', packages=['fakepackage1', 'fakepackage2']) + mock_bu.do_post_inst.assert_called_once_with('/tmp/imgdir') + signal_calls = mock_bu.send_signal_to_chrooted_processes.call_args_list + self.assertEqual([mock.call('/tmp/imgdir', signal.SIGTERM), + mock.call('/tmp/imgdir', signal.SIGKILL)], + signal_calls) + mock_sleep.assert_called_once_with(2) + mock_fu.umount_fs.assert_called_once_with('/tmp/imgdir/proc') + mock_umount_target.assert_called_once_with('/tmp/imgdir', pseudo=False) + self.assertEqual([mock.call('/dev/loop0'), mock.call('/dev/loop1')], + mock_bu.deattach_loop.call_args_list) + self.assertEqual([mock.call('/tmp/img'), mock.call('/tmp/img-boot')], + mock_bu.shrink_sparse_file.call_args_list) + self.assertEqual([mock.call('/tmp/img'), + mock.call('/fake/img.img.gz'), + mock.call('/tmp/img-boot'), + mock.call('/fake/img-boot.img.gz')], + mock_os.path.getsize.call_args_list) + self.assertEqual([mock.call('/tmp/img', 20), + mock.call('/fake/img.img.gz', 2), + mock.call('/tmp/img-boot', 10), + mock.call('/fake/img-boot.img.gz', 1)], + mock_utils.calculate_md5.call_args_list) + self.assertEqual([mock.call('/tmp/img', 'gzip'), + mock.call('/tmp/img-boot', 'gzip')], + mock_bu.containerize.call_args_list) + mock_open.assert_called_once_with('/fake/img.yaml', 'w') + self.assertEqual( + [mock.call('/tmp/img.gz', '/fake/img.img.gz'), + mock.call('/tmp/img-boot.gz', '/fake/img-boot.img.gz')], + mock_shutil_move.call_args_list) + + metadata = {} + for repo in self.mgr.driver.operating_system.repos: + metadata.setdefault('repos', []).append({ + 'type': 'deb', + 'name': repo.name, + 'uri': repo.uri, + 'suite': repo.suite, + 'section': repo.section, + 'priority': repo.priority, + 'meta': repo.meta}) + metadata['packages'] = self.mgr.driver.operating_system.packages + metadata['images'] = [ + { + 'raw_md5': md5_side[0], + 'raw_size': getsize_side[0], + 'raw_name': None, + 'container_name': + os.path.basename( + self.mgr.driver.image_scheme.images[0].uri.split( + 'file://', 1)[1]), + 'container_md5': md5_side[1], + 'container_size': getsize_side[1], + 'container': self.mgr.driver.image_scheme.images[0].container, + 'format': self.mgr.driver.image_scheme.images[0].format + }, + { + 'raw_md5': md5_side[2], + 'raw_size': getsize_side[2], + 'raw_name': None, + 'container_name': + os.path.basename( + self.mgr.driver.image_scheme.images[1].uri.split( + 'file://', 1)[1]), + 'container_md5': md5_side[3], + 'container_size': getsize_side[3], + 'container': self.mgr.driver.image_scheme.images[1].container, + 'format': self.mgr.driver.image_scheme.images[1].format + } + ] + mock_yaml_dump.assert_called_once_with(metadata, stream=mock_open()) diff --git a/fuel_agent/tests/test_nailgun.py b/fuel_agent/tests/test_nailgun.py index e254e61..4c8b2ea 100644 --- a/fuel_agent/tests/test_nailgun.py +++ b/fuel_agent/tests/test_nailgun.py @@ -616,7 +616,8 @@ class TestNailgun(test_base.BaseTestCase): @mock.patch.object(utils, 'init_http_request') @mock.patch.object(hu, 'list_block_devices') def test_image_scheme_with_checksums(self, mock_lbd, mock_http_req): - fake_image_meta = [{'/': {'md5': 'fakeroot', 'size': 1}}] + fake_image_meta = {'images': [{'raw_md5': 'fakeroot', 'raw_size': 1, + 'container_name': 'fake_image.img.gz'}]} prop_mock = mock.PropertyMock(return_value=yaml.dump(fake_image_meta)) type(mock_http_req.return_value).text = prop_mock mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE @@ -646,8 +647,9 @@ class TestNailgun(test_base.BaseTestCase): expected_images[i].format) self.assertEqual(img.container, expected_images[i].container) - self.assertEqual(img.size, fake_image_meta[0]['/']['size']) - self.assertEqual(img.md5, fake_image_meta[0]['/']['md5']) + self.assertEqual( + img.size, fake_image_meta['images'][0]['raw_size']) + self.assertEqual(img.md5, fake_image_meta['images'][0]['raw_md5']) def test_getlabel(self): self.assertEqual('', self.drv._getlabel(None)) diff --git a/fuel_agent/tests/test_nailgun_build_image.py b/fuel_agent/tests/test_nailgun_build_image.py new file mode 100644 index 0000000..635cbd4 --- /dev/null +++ b/fuel_agent/tests/test_nailgun_build_image.py @@ -0,0 +1,244 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import os +import six +from six.moves.urllib.parse import urlsplit + +from oslotest import base as test_base + +from fuel_agent.drivers.nailgun import NailgunBuildImage +from fuel_agent import errors +from fuel_agent import objects + +DEFAULT_TRUSTY_PACKAGES = [ + "acl", + "anacron", + "bash-completion", + "bridge-utils", + "bsdmainutils", + "build-essential", + "cloud-init", + "curl", + "daemonize", + "debconf-utils", + "gdisk", + "grub-pc", + "linux-firmware", + "linux-firmware-nonfree", + "linux-headers-generic-lts-trusty", + "linux-image-generic-lts-trusty", + "lvm2", + "mcollective", + "mdadm", + "nailgun-agent", + "nailgun-mcagents", + "nailgun-net-check", + "ntp", + "openssh-client", + "openssh-server", + "puppet", + "python-amqp", + "ruby-augeas", + "ruby-ipaddress", + "ruby-json", + "ruby-netaddr", + "ruby-openstack", + "ruby-shadow", + "ruby-stomp", + "telnet", + "ubuntu-minimal", + "ubuntu-standard", + "uuid-runtime", + "vim", + "virt-what", + "vlan", +] + +REPOS_SAMPLE = [ + { + "name": "ubuntu", + "section": "main universe multiverse", + "uri": "http://archive.ubuntu.com/ubuntu/", + "priority": None, + "suite": "trusty", + "type": "deb" + }, + { + "name": "mos", + "section": "main restricted", + "uri": "http://10.20.0.2:8080/2014.2-6.1/ubuntu/x86_64", + "priority": 1050, + "suite": "mos6.1", + "type": "deb" + } +] + +IMAGE_DATA_SAMPLE = { + "/boot": { + "container": "gzip", + "uri": "http://10.20.0.2:8080/path/to/img-boot.img.gz", + "format": "ext2" + }, + "/": { + "container": "gzip", + "uri": "http://10.20.0.2:8080/path/to/img.img.gz", + "format": "ext4" + } +} + + +class TestNailgunBuildImage(test_base.BaseTestCase): + + def test_default_trusty_packages(self): + self.assertEqual(NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES, + DEFAULT_TRUSTY_PACKAGES) + + @mock.patch.object(NailgunBuildImage, '__init__') + def test_parse_operating_system_error_bad_codename(self, mock_init): + mock_init.return_value = None + driver = NailgunBuildImage() + driver.data = {'codename': 'not-trusty'} + self.assertRaises(errors.WrongInputDataError, + driver.parse_operating_system) + + @mock.patch('fuel_agent.objects.Ubuntu') + @mock.patch.object(NailgunBuildImage, '__init__') + def test_parse_operating_system_packages_given(self, mock_init, mock_ub): + mock_init.return_value = None + data = { + 'repos': [], + 'codename': 'trusty', + 'packages': ['pack'] + } + driver = NailgunBuildImage() + driver.data = data + mock_ub_instance = mock_ub.return_value + mock_ub_instance.packages = data['packages'] + driver.parse_operating_system() + mock_ub.assert_called_once_with(repos=[], packages=data['packages']) + self.assertEqual(driver.operating_system.packages, data['packages']) + + @mock.patch('fuel_agent.objects.Ubuntu') + @mock.patch.object(NailgunBuildImage, '__init__') + def test_parse_operating_system_packages_not_given( + self, mock_init, mock_ub): + mock_init.return_value = None + data = { + 'repos': [], + 'codename': 'trusty' + } + driver = NailgunBuildImage() + driver.data = data + mock_ub_instance = mock_ub.return_value + mock_ub_instance.packages = NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES + driver.parse_operating_system() + mock_ub.assert_called_once_with( + repos=[], packages=NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES) + self.assertEqual(driver.operating_system.packages, + NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES) + + @mock.patch('fuel_agent.objects.DEBRepo') + @mock.patch('fuel_agent.objects.Ubuntu') + @mock.patch.object(NailgunBuildImage, '__init__') + def test_parse_operating_system_repos(self, mock_init, mock_ub, mock_deb): + mock_init.return_value = None + data = { + 'repos': REPOS_SAMPLE, + 'codename': 'trusty' + } + driver = NailgunBuildImage() + driver.data = data + + mock_deb_expected_calls = [] + repos = [] + for r in REPOS_SAMPLE: + kwargs = { + 'name': r['name'], + 'uri': r['uri'], + 'suite': r['suite'], + 'section': r['section'], + 'priority': r['priority'] + } + mock_deb_expected_calls.append(mock.call(**kwargs)) + repos.append(objects.DEBRepo(**kwargs)) + driver.parse_operating_system() + mock_ub_instance = mock_ub.return_value + mock_ub_instance.repos = repos + mock_ub.assert_called_once_with( + repos=repos, packages=NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES) + self.assertEqual(mock_deb_expected_calls, + mock_deb.call_args_list[:len(REPOS_SAMPLE)]) + self.assertEqual(driver.operating_system.repos, repos) + + @mock.patch('fuel_agent.drivers.nailgun.objects.Loop') + @mock.patch('fuel_agent.objects.Image') + @mock.patch('fuel_agent.objects.Fs') + @mock.patch('fuel_agent.objects.PartitionScheme') + @mock.patch('fuel_agent.objects.ImageScheme') + @mock.patch.object(NailgunBuildImage, '__init__') + def test_parse_schemes( + self, mock_init, mock_imgsch, mock_partsch, + mock_fs, mock_img, mock_loop): + mock_init.return_value = None + data = { + 'image_data': IMAGE_DATA_SAMPLE, + 'output': '/some/local/path', + } + driver = NailgunBuildImage() + driver.data = data + driver.parse_schemes() + + mock_fs_expected_calls = [] + mock_img_expected_calls = [] + images = [] + fss = [] + data_length = len(data['image_data'].keys()) + for mount, image in six.iteritems(data['image_data']): + filename = os.path.basename(urlsplit(image['uri']).path) + img_kwargs = { + 'uri': 'file://' + os.path.join(data['output'], filename), + 'format': image['format'], + 'container': image['container'], + 'target_device': None + } + mock_img_expected_calls.append(mock.call(**img_kwargs)) + images.append(objects.Image(**img_kwargs)) + + fs_kwargs = { + 'device': None, + 'mount': mount, + 'fs_type': image['format'] + } + mock_fs_expected_calls.append(mock.call(**fs_kwargs)) + fss.append(objects.Fs(**fs_kwargs)) + + if mount == '/': + metadata_filename = filename.split('.', 1)[0] + '.yaml' + + mock_imgsch_instance = mock_imgsch.return_value + mock_imgsch_instance.images = images + mock_partsch_instance = mock_partsch.return_value + mock_partsch_instance.fss = fss + + self.assertEqual( + driver.metadata_uri, 'file://' + os.path.join( + data['output'], metadata_filename)) + self.assertEqual(mock_img_expected_calls, + mock_img.call_args_list[:data_length]) + self.assertEqual(mock_fs_expected_calls, + mock_fs.call_args_list[:data_length]) + self.assertEqual(driver.image_scheme.images, images) + self.assertEqual(driver.partition_scheme.fss, fss) diff --git a/fuel_agent/tests/test_utils.py b/fuel_agent/tests/test_utils.py index 24335ad..6289d34 100644 --- a/fuel_agent/tests/test_utils.py +++ b/fuel_agent/tests/test_utils.py @@ -34,7 +34,7 @@ class ExecuteTestCase(testtools.TestCase): def setUp(self): super(ExecuteTestCase, self).setUp() fake_driver = stevedore.extension.Extension('fake_driver', None, None, - 'fake_obj') + mock.MagicMock) self.drv_manager = stevedore.driver.DriverManager.make_test_instance( fake_driver) @@ -64,7 +64,8 @@ class ExecuteTestCase(testtools.TestCase): @mock.patch('stevedore.driver.DriverManager') def test_get_driver(self, mock_drv_manager): mock_drv_manager.return_value = self.drv_manager - self.assertEqual('fake_obj', utils.get_driver('fake_driver')) + self.assertEqual(mock.MagicMock.__name__, + utils.get_driver('fake_driver').__name__) @mock.patch('jinja2.Environment') @mock.patch('jinja2.FileSystemLoader') @@ -136,3 +137,26 @@ class ExecuteTestCase(testtools.TestCase): mock_req.side_effect = requests.exceptions.ConnectionError() self.assertRaises(errors.HttpUrlConnectionError, utils.init_http_request, 'fake_url') + + @mock.patch('fuel_agent.utils.utils.os.makedirs') + @mock.patch('fuel_agent.utils.utils.os.path.isdir', return_value=False) + def test_makedirs_if_not_exists(self, mock_isdir, mock_makedirs): + utils.makedirs_if_not_exists('/fake/path') + mock_isdir.assert_called_once_with('/fake/path') + mock_makedirs.assert_called_once_with('/fake/path', mode=0o755) + + @mock.patch('fuel_agent.utils.utils.os.makedirs') + @mock.patch('fuel_agent.utils.utils.os.path.isdir', return_value=False) + def test_makedirs_if_not_exists_mode_given( + self, mock_isdir, mock_makedirs): + utils.makedirs_if_not_exists('/fake/path', mode=0o000) + mock_isdir.assert_called_once_with('/fake/path') + mock_makedirs.assert_called_once_with('/fake/path', mode=0o000) + + @mock.patch('fuel_agent.utils.utils.os.makedirs') + @mock.patch('fuel_agent.utils.utils.os.path.isdir', return_value=True) + def test_makedirs_if_not_exists_already_exists( + self, mock_isdir, mock_makedirs): + utils.makedirs_if_not_exists('/fake/path') + mock_isdir.assert_called_once_with('/fake/path') + self.assertEqual(mock_makedirs.mock_calls, []) diff --git a/fuel_agent/utils/utils.py b/fuel_agent/utils/utils.py index eb350d8..a8e181a 100644 --- a/fuel_agent/utils/utils.py +++ b/fuel_agent/utils/utils.py @@ -135,8 +135,11 @@ def B2MiB(b, ceil=True): def get_driver(name): - return stevedore.driver.DriverManager( + LOG.debug('Trying to get driver: fuel_agent.drivers.%s', name) + driver = stevedore.driver.DriverManager( namespace='fuel_agent.drivers', name=name).driver + LOG.debug('Found driver: %s', driver.__name__) + return driver def render_and_save(tmpl_dir, tmpl_names, tmpl_data, file_name): @@ -200,3 +203,12 @@ def init_http_request(url, byte_range=0): 'Exceeded maximum http request retries for %s' % url) response_obj.raise_for_status() return response_obj + + +def makedirs_if_not_exists(path, mode=0o755): + """Create directory if it does not exist + :param path: Directory path + :param mode: Directory mode (Default: 0o755) + """ + if not os.path.isdir(path): + os.makedirs(path, mode=mode) diff --git a/requirements.txt b/requirements.txt index 50dba68..86256c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ eventlet>=0.13.0 iso8601>=0.1.9 jsonschema>=2.3.0 oslo.config>=1.2.0 -oslo.serialization>=1.0.0 +oslo.serialization>=1.4.0 six>=1.5.2 pbr>=0.7.0 Jinja2 diff --git a/setup.cfg b/setup.cfg index 589bfb5..741e6c9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,14 +14,17 @@ packages = [entry_points] console_scripts = + # TODO(kozhukalov): rename entry point provision = fuel_agent.cmd.agent:provision - partition = fuel_agent.cmd.agent:partition - configdrive = fuel_agent.cmd.agent:configdrive - copyimage = fuel_agent.cmd.agent:copyimage - bootloader = fuel_agent.cmd.agent:bootloader + fa_partition = fuel_agent.cmd.agent:partition + fa_configdrive = fuel_agent.cmd.agent:configdrive + fa_copyimage = fuel_agent.cmd.agent:copyimage + fa_bootloader = fuel_agent.cmd.agent:bootloader + fa_build_image = fuel_agent.cmd.agent:build_image fuel_agent.drivers = nailgun = fuel_agent.drivers.nailgun:Nailgun + nailgun_build_image = fuel_agent.drivers.nailgun:NailgunBuildImage [pbr] autodoc_index_modules = True