diff --git a/striker/common/utils.py b/striker/common/utils.py new file mode 100644 index 0000000..fbca7b5 --- /dev/null +++ b/striker/common/utils.py @@ -0,0 +1,58 @@ +# Copyright 2014 Rackspace +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS +# IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +import os +import time + + +def canonicalize_path(cwd, path): + """ + Canonicalizes a path relative to a given working directory. + + :param cwd: The working directory to interpret ``path`` relative + to. + :param path: The path to canonicalize. If relative, it will be + interpreted relative to ``cwd``. + + :returns: The absolute path. + """ + + if not os.path.isabs(path): + path = os.path.join(cwd, path) + + return os.path.abspath(path) + + +def backoff(max_tries): + """ + A generator to perform simplified exponential backoff. Yields up + to the specified number of times, performing a ``time.sleep()`` + with an exponentially increasing sleep time (starting at 1 second) + between each trial. Yields the (0-based) trial number. + + :param max_tries: The maximum number of tries to attempt. + """ + + # How much time will we sleep next time? + sleep = 1 + + for i in range(max_tries): + # Yield the trial number + yield i + + # We've re-entered the loop; sleep, then increment the sleep + # time + time.sleep(sleep) + sleep <<= 1 diff --git a/striker/core/context.py b/striker/core/context.py new file mode 100644 index 0000000..4eb7471 --- /dev/null +++ b/striker/core/context.py @@ -0,0 +1,123 @@ +# Copyright 2014 Rackspace +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS +# IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +from striker.core import environment + + +class Context(object): + """ + Execution context. Objects of this class contain all the basic + configuration data needed to perform a task. + """ + + def __init__(self, workspace, config, logger, + debug=False, dry_run=False, **extras): + """ + Initialize a ``Context`` object. + + :param workspace: The name of a temporary working directory. + The directory must exist. + :param config: An object containing configuration data. The + object should support read-only attribute-style + access to the configuration settings. + :param logger: An object compatible with ``logging.Logger``. + This will be used by all consumers of the + ``Context`` to emit logging information. + :param debug: A boolean, defaulting to ``False``, indicating + whether debugging mode is active. + :param dry_run: A boolean, defaulting to ``False``, indicating + whether permanent changes should be effected. + This should be used to control whether files + are uploaded, for instance. + :param extras: Keyword arguments specifying additional data to + be stored in the context. This could be, for + instance, account data. + """ + + # Store basic context data + self.workspace = workspace + self.config = config + self.logger = logger + self.debug = debug + self.dry_run = dry_run + + # Extra data--things like accounts + self._extras = extras + + # Environment + self._environ = None + + def __getattr__(self, name): + """ + Provides access to the extra data specified to the constructor. + + :param name: The name of the extra datum to retrieve. + + :returns: The value of the extra datum. + """ + + if name not in self._extras: + raise AttributeError("'%s' object has no attribute '%s'" % + (self.__class__.__name__, name)) + + return self._extras[name] + + @property + def environ(self): + """ + Access the environment. The environment is a dictionary of + environment variables, but it is also a callable that can be + used to invoke shell commands. + + :param cmd: The command to execute, as either a bare string or + a list of arguments. If a string, it will be + split into a list using ``shlex.split()``. Note + that use of bare strings for this argument is + discouraged. + :param capture_output: If ``True``, standard input and output + will be captured, and will be available + in the result. Defaults to ``False``. + Note that this is treated as implicitly + ``True`` if the ``retry`` parameter is + provided. + :param cwd: Gives an alternate working directory from which to + run the command. + :param do_raise: If ``True`` (the default), an execution + failure will raise an exception. + :param retry: If provided, must be a callable taking one + argument. Will be called with an instance of + ``ExecResult``, and can return ``True`` to + indicate that the call should be retried. + Retries are performed with an exponential + backoff controlled by ``max_tries``. + :param max_tries: The maximum number of tries to perform + before giving up, if ``retry`` is specified. + Retries are performed with an exponential + backoff: the first try is performed + immediately, and subsequent tries occur + after a sleep time that starts at one second + and is doubled for each try. + + :returns: An ``ExecResult`` object containing the results of + the execution. If the return code was non-zero and + ``do_raise`` is ``True``, this is the object that + will be raised. + """ + + # Construct the environment if necessary + if self._environ is None: + self._environ = environment.Environment(self.logger) + + return self._environ diff --git a/striker/core/environment.py b/striker/core/environment.py new file mode 100644 index 0000000..d43e7ee --- /dev/null +++ b/striker/core/environment.py @@ -0,0 +1,301 @@ +# Copyright 2014 Rackspace +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS +# IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +import os +import shlex +import shutil +import subprocess + +import six + +from striker.common import utils + + +class ExecResult(Exception): + """ + Encapsulate the results of calling a command. This class extends + ``Exception`` so that it can be raised in the event of a command + failure. The command executed is available in both list (``cmd``) + and plain text (``cmd_text``) forms. If the command is executed + with ``capture_output``, the standard output (``stdout``) and + standard error (``stderr``) streams will also be available. The + command return code is available in the ``return_code`` attribute. + """ + + def __init__(self, cmd, stdout, stderr, return_code): + """ + Initialize an ``ExecResult``. + + :param cmd: The command, in list format. + :param stdout: The standard output from the command execution. + :param stderr: The standard error from the command execution. + :param return_code: The return code from executing the + command. + """ + + # Store all the data + self.cmd = cmd + self.stdout = stdout + self.stderr = stderr + self.return_code = return_code + + # Form the command text + comps = [] + for comp in cmd: + # Determine if the component needs quoting + if ' ' in comp or '"' in comp or "'" in comp: + # Escape any double-quotes + parts = comp.split('"') + comp = '"%s"' % '\\"'.join(parts) + + comps.append(comp) + + # Save the command text + self.cmd_text = ' '.join(comps) + + # Formulate the message + if return_code: + msg = ("'%s' failed with return code %s" % + (self.cmd_text, return_code)) + elif stderr: + msg = "'%s' said: %s" % (self.cmd_text, stderr) + elif stdout: + msg = "'%s' said: %s" % (self.cmd_text, stdout) + else: + msg = "'%s' succeeded" % (self.cmd_text,) + + # Initialize ourselves as an exception + super(ExecResult, self).__init__(msg) + + def __nonzero__(self): + """ + Allows conversion of an ``ExecResult`` to boolean, based on + the command return code. If the return code was 0, the object + will be considered ``True``; otherwise, the object will be + considered ``False``. + + :returns: ``True`` if the command succeeded, ``False`` + otherwise. + """ + + return not bool(self.return_code) + __bool__ = __nonzero__ + + +class Environment(dict): + """ + Describes an environment that can be used for execution of + subprocesses. Virtual environments can be created by calling the + ``create_venv()`` method, which returns an independent instance of + ``Environment``. + """ + + def __init__(self, logger, environ=None, cwd=None, venv_home=None): + """ + Initialize a new ``Environment``. + + :param logger: An object compatible with ``logging.Logger``. + This will be used to emit logging information. + :param environ: A dictionary containing the environment + variables. If not given, ``os.environ`` will + be used. + :param cwd: The working directory to use. If relative, will + be interpreted relative to the current working + directory. If not given, the current working + directory will be used. + :param venv_home: The home directory for the virtual + environment. + """ + + super(Environment, self).__init__(environ or os.environ) + + # Save the logger + self.logger = logger + + # Change to the desired working directory, then save the full + # path to it + self.cwd = os.getcwd() + if cwd: + self.chdir(cwd) + + # Save the virtual environment home + self.venv_home = venv_home + + def __call__(self, cmd, capture_output=False, cwd=None, do_raise=True, + retry=None, max_tries=5): + """ + Execute a command in the context of this environment. + + :param cmd: The command to execute, as either a bare string or + a list of arguments. If a string, it will be + split into a list using ``shlex.split()``. Note + that use of bare strings for this argument is + discouraged. + :param capture_output: If ``True``, standard input and output + will be captured, and will be available + in the result. Defaults to ``False``. + Note that this is treated as implicitly + ``True`` if the ``retry`` parameter is + provided. + :param cwd: Gives an alternate working directory from which to + run the command. + :param do_raise: If ``True`` (the default), an execution + failure will raise an exception. + :param retry: If provided, must be a callable taking one + argument. Will be called with an instance of + ``ExecResult``, and can return ``True`` to + indicate that the call should be retried. + Retries are performed with an exponential + backoff controlled by ``max_tries``. + :param max_tries: The maximum number of tries to perform + before giving up, if ``retry`` is specified. + Retries are performed with an exponential + backoff: the first try is performed + immediately, and subsequent tries occur + after a sleep time that starts at one second + and is doubled for each try. + + :returns: An ``ExecResult`` object containing the results of + the execution. If the return code was non-zero and + ``do_raise`` is ``True``, this is the object that + will be raised. + """ + + # Sanity-check arguments + if not retry or max_tries < 1: + max_tries = 1 + + # Determine the working directory to use + cwd = utils.canonicalize_path(self.cwd, cwd) if cwd else self.cwd + + # Turn simple strings into lists of tokens + if isinstance(cmd, six.string_types): + self.logger.debug("Notice: splitting command string '%s'" % + cmd) + cmd = shlex.split(cmd) + + self.logger.debug("Executing command: %r (cwd %s)" % (cmd, cwd)) + + # Prepare the keyword arguments for the Popen call + kwargs = { + 'env': self, + 'cwd': cwd, + 'close_fds': True, + } + + # Set up stdout and stderr + if capture_output or (retry and max_tries > 1): + kwargs.update({ + 'stdout': subprocess.PIPE, + 'stderr': subprocess.PIPE, + }) + + # Perform the tries in a loop + for trial in utils.backoff(max_tries): + if trial: + self.logger.warn("Failure caught; retrying command " + "(try #%d)" % (trial + 1)) + + # Call the command + child = subprocess.Popen(cmd, **kwargs) + stdout, stderr = child.communicate() + result = ExecResult(cmd, stdout, stderr, child.returncode) + + # Check if we need to retry + if retry and not result and retry(result): + continue + + break + else: + # Just log a warning that we couldn't retry + self.logger.warn("Unable to retry: too many attempts") + + # Raise an exception if requested + if not result and do_raise: + raise result + + return result + + def chdir(self, path): + """ + Change the working directory. + + :param path: The path to change to. If relative, will be + interpreted relative to the current working + directory. + + :returns: The new working directory. + """ + + self.cwd = utils.canonicalize_path(self.cwd, path) + + return self.cwd + + def create_venv(self, path, rebuild=False, **kwargs): + """ + Create a new, bare virtual environment rooted at the given + directory. No packages will be installed, except what + ``virtualenv`` installs. Returns a new ``Environment`` set up + for the new virtual environment, with the working directory + set to be the same as the virtual environment directory. Any + keyword arguments will override system environment variables + in the new ``Environment`` object. + + :param path: The path to create the virtual environment in. + If relative, will be interpreted relative to the + current working directory. + :param rebuild: If ``True``, the virtual environment will be + rebuilt even if it already exists. If + ``False`` (the default), the virtual + environment will only be rebuilt if it doesn't + already exist. + :returns: A new ``Environment`` object. + """ + + # Determine the new virtual environment path + path = utils.canonicalize_path(self.cwd, path) + + self.logger.debug("Preparing virtual environment %s" % path) + + # Check if we need to rebuild the virtual environment + if os.path.exists(path): + if rebuild: + # Blow away the old tree + self.logger.info("Destroying old virtual environment %s" % + path) + shutil.rmtree(path) + else: + self.logger.info("Using existing virtual environment %s" % + path) + else: + # We'll need to create it + rebuild = True + + # Create the new virtual environment + if rebuild: + self.logger.info("Creating virtual environment %s" % path) + self(['virtualenv', path]) + + # Set up the environment variables that are needed + kwargs.setdefault('VIRTUAL_ENV', path) + bindir = os.path.join(path, 'bin') + kwargs.setdefault('PATH', '%s%s%s' % + (bindir, os.pathsep, self['PATH'])) + + # Set up and return the new Environment + new_env = self.__class__(self.logger, environ=self, cwd=path, + venv_home=path) + new_env.update(kwargs) + return new_env diff --git a/tests/unit/common/test_utils.py b/tests/unit/common/test_utils.py new file mode 100644 index 0000000..7f5ccc1 --- /dev/null +++ b/tests/unit/common/test_utils.py @@ -0,0 +1,74 @@ +# Copyright 2014 Rackspace +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS +# IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +import unittest + +import mock + +from striker.common import utils + +import tests + + +class CanonicalizePathTest(unittest.TestCase): + @mock.patch('os.path.isabs', tests.fake_isabs) + @mock.patch('os.path.join', tests.fake_join) + @mock.patch('os.path.abspath', tests.fake_abspath) + def test_absolute(self): + result = utils.canonicalize_path('/foo/bar', '/bar/baz') + + self.assertEqual(result, '/bar/baz') + + @mock.patch('os.path.isabs', tests.fake_isabs) + @mock.patch('os.path.join', tests.fake_join) + @mock.patch('os.path.abspath', tests.fake_abspath) + def test_relative(self): + result = utils.canonicalize_path('/foo/bar', 'bar/baz') + + self.assertEqual(result, '/foo/bar/bar/baz') + + @mock.patch('os.path.isabs', tests.fake_isabs) + @mock.patch('os.path.join', tests.fake_join) + @mock.patch('os.path.abspath', tests.fake_abspath) + def test_relative_with_cwd(self): + result = utils.canonicalize_path('/foo/bar', './baz') + + self.assertEqual(result, '/foo/bar/baz') + + @mock.patch('os.path.isabs', tests.fake_isabs) + @mock.patch('os.path.join', tests.fake_join) + @mock.patch('os.path.abspath', tests.fake_abspath) + def test_relative_with_parent(self): + result = utils.canonicalize_path('/foo/bar', '../baz') + + self.assertEqual(result, '/foo/baz') + + +class BackoffTest(unittest.TestCase): + @mock.patch('time.sleep') + def test_backoff(self, mock_sleep): + max_tries = 5 + + for i, trial in enumerate(utils.backoff(max_tries)): + self.assertEqual(i, trial) + + if i: + mock_sleep.assert_called_once_with(1 << (i - 1)) + else: + self.assertFalse(mock_sleep.called) + + mock_sleep.reset_mock() + + self.assertEqual(i, max_tries - 1) diff --git a/tests/unit/core/test_context.py b/tests/unit/core/test_context.py new file mode 100644 index 0000000..283f08a --- /dev/null +++ b/tests/unit/core/test_context.py @@ -0,0 +1,79 @@ +# Copyright 2014 Rackspace +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS +# IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +import unittest + +import mock + +from striker.core import context +from striker.core import environment + + +class ContextTest(unittest.TestCase): + def test_init_base(self): + ctxt = context.Context('/path/to/workspace', 'config', 'logger') + + self.assertEqual(ctxt.workspace, '/path/to/workspace') + self.assertEqual(ctxt.config, 'config') + self.assertEqual(ctxt.logger, 'logger') + self.assertEqual(ctxt.debug, False) + self.assertEqual(ctxt.dry_run, False) + self.assertEqual(ctxt._extras, {}) + self.assertEqual(ctxt._environ, None) + + def test_init_alt(self): + ctxt = context.Context('/path/to/workspace', 'config', 'logger', + debug=True, dry_run=True, accounts='accounts', + other='other') + + self.assertEqual(ctxt.workspace, '/path/to/workspace') + self.assertEqual(ctxt.config, 'config') + self.assertEqual(ctxt.logger, 'logger') + self.assertEqual(ctxt.debug, True) + self.assertEqual(ctxt.dry_run, True) + self.assertEqual(ctxt._extras, { + 'accounts': 'accounts', + 'other': 'other', + }) + self.assertEqual(ctxt._environ, None) + + def test_getattr_exists(self): + ctxt = context.Context('/path/to/workspace', 'config', 'logger', + attr='value') + + self.assertEqual(ctxt.attr, 'value') + + def test_getattr_noexist(self): + ctxt = context.Context('/path/to/workspace', 'config', 'logger', + attr='value') + + self.assertRaises(AttributeError, lambda: ctxt.other) + + @mock.patch.object(environment, 'Environment', return_value='environ') + def test_environ_cached(self, mock_Environment): + ctxt = context.Context('/path/to/workspace', 'config', 'logger') + ctxt._environ = 'cached' + + self.assertEqual(ctxt.environ, 'cached') + self.assertEqual(ctxt._environ, 'cached') + self.assertFalse(mock_Environment.called) + + @mock.patch.object(environment, 'Environment', return_value='environ') + def test_environ_uncached(self, mock_Environment): + ctxt = context.Context('/path/to/workspace', 'config', 'logger') + + self.assertEqual(ctxt.environ, 'environ') + self.assertEqual(ctxt._environ, 'environ') + mock_Environment.assert_called_once_with('logger') diff --git a/tests/unit/core/test_environment.py b/tests/unit/core/test_environment.py new file mode 100644 index 0000000..457594d --- /dev/null +++ b/tests/unit/core/test_environment.py @@ -0,0 +1,667 @@ +# Copyright 2014 Rackspace +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS +# IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +import os +import subprocess +import unittest + +import mock + +from striker.common import utils +from striker.core import environment + +import tests + + +class ExecResultTest(unittest.TestCase): + def test_init_success(self): + cmd = ['arg1', 'arg2 space', 'arg3"double', "arg4'single", 'arg5'] + cmd_text = 'arg1 "arg2 space" "arg3\\"double" "arg4\'single" arg5' + result = environment.ExecResult(cmd, None, None, 0) + + self.assertEqual(result.cmd, cmd) + self.assertEqual(result.cmd_text, cmd_text) + self.assertEqual(result.stdout, None) + self.assertEqual(result.stderr, None) + self.assertEqual(result.return_code, 0) + self.assertEqual(str(result), "'%s' succeeded" % cmd_text) + + def test_init_stdout(self): + cmd = ['arg1', 'arg2 space', 'arg3"double', "arg4'single", 'arg5'] + cmd_text = 'arg1 "arg2 space" "arg3\\"double" "arg4\'single" arg5' + result = environment.ExecResult(cmd, 'output', None, 0) + + self.assertEqual(result.cmd, cmd) + self.assertEqual(result.cmd_text, cmd_text) + self.assertEqual(result.stdout, 'output') + self.assertEqual(result.stderr, None) + self.assertEqual(result.return_code, 0) + self.assertEqual(str(result), "'%s' said: output" % cmd_text) + + def test_init_stderr(self): + cmd = ['arg1', 'arg2 space', 'arg3"double', "arg4'single", 'arg5'] + cmd_text = 'arg1 "arg2 space" "arg3\\"double" "arg4\'single" arg5' + result = environment.ExecResult(cmd, 'output', 'error', 0) + + self.assertEqual(result.cmd, cmd) + self.assertEqual(result.cmd_text, cmd_text) + self.assertEqual(result.stdout, 'output') + self.assertEqual(result.stderr, 'error') + self.assertEqual(result.return_code, 0) + self.assertEqual(str(result), "'%s' said: error" % cmd_text) + + def test_init_failure(self): + cmd = ['arg1', 'arg2 space', 'arg3"double', "arg4'single", 'arg5'] + cmd_text = 'arg1 "arg2 space" "arg3\\"double" "arg4\'single" arg5' + result = environment.ExecResult(cmd, 'output', 'error', 5) + + self.assertEqual(result.cmd, cmd) + self.assertEqual(result.cmd_text, cmd_text) + self.assertEqual(result.stdout, 'output') + self.assertEqual(result.stderr, 'error') + self.assertEqual(result.return_code, 5) + self.assertEqual(str(result), "'%s' failed with return code 5" % + cmd_text) + + def test_true(self): + result = environment.ExecResult(['cmd'], None, None, 0) + + self.assertTrue(result) + + def test_false(self): + result = environment.ExecResult(['cmd'], None, None, 1) + + self.assertFalse(result) + + +class EnvironmentTest(unittest.TestCase): + @mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2') + @mock.patch.object(os, 'getcwd', return_value='/some/path') + @mock.patch.object(environment.Environment, 'chdir') + def test_init_base(self, mock_chdir, mock_getcwd): + env = environment.Environment('logger') + + self.assertEqual(env, {'TEST_VAR1': '1', 'TEST_VAR2': '2'}) + self.assertEqual(env.logger, 'logger') + self.assertEqual(env.cwd, '/some/path') + self.assertEqual(env.venv_home, None) + self.assertFalse(mock_chdir.called) + + @mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2') + @mock.patch.object(os, 'getcwd', return_value='/some/path') + @mock.patch.object(environment.Environment, 'chdir') + def test_init_alt(self, mock_chdir, mock_getcwd): + environ = { + 'TEST_VAR3': '3', + 'TEST_VAR4': '4', + } + env = environment.Environment('logger', environ, '/other/path', + '/venv/home') + + self.assertEqual(env, environ) + self.assertEqual(env.logger, 'logger') + self.assertEqual(env.cwd, '/some/path') + self.assertEqual(env.venv_home, '/venv/home') + mock_chdir.assert_called_once_with('/other/path') + + @mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2') + @mock.patch.object(os.path, 'join', tests.fake_join) + @mock.patch.object(os, 'pathsep', ':') + @mock.patch.object(os, 'getcwd', return_value='/some/path') + @mock.patch.object(environment.Environment, 'chdir') + @mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path') + @mock.patch.object(utils, 'backoff', return_value=[0]) + @mock.patch.object(subprocess, 'Popen', return_value=mock.Mock(**{ + 'returncode': 0, + 'communicate.return_value': (None, None), + })) + def test_call_basic(self, mock_Popen, mock_backoff, mock_canonicalize_path, + mock_chdir, mock_getcwd): + logger = mock.Mock() + env = environment.Environment(logger) + + result = env(['test', 'one', 'two']) + + self.assertEqual(result.cmd, ['test', 'one', 'two']) + self.assertEqual(result.stdout, None) + self.assertEqual(result.stderr, None) + self.assertEqual(result.return_code, 0) + self.assertFalse(mock_canonicalize_path.called) + mock_backoff.assert_called_once_with(1) + mock_Popen.assert_called_once_with( + ['test', 'one', 'two'], env=env, cwd='/some/path', close_fds=True) + logger.assert_has_calls([ + mock.call.debug( + "Executing command: ['test', 'one', 'two'] (cwd /some/path)"), + ]) + self.assertEqual(len(logger.method_calls), 1) + + @mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2') + @mock.patch.object(os, 'getcwd', return_value='/some/path') + @mock.patch.object(environment.Environment, 'chdir') + @mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path') + @mock.patch.object(utils, 'backoff', return_value=[0]) + @mock.patch.object(subprocess, 'Popen', return_value=mock.Mock(**{ + 'returncode': 0, + 'communicate.return_value': (None, None), + })) + def test_call_string(self, mock_Popen, mock_backoff, + mock_canonicalize_path, mock_chdir, mock_getcwd): + logger = mock.Mock() + env = environment.Environment(logger) + + result = env("test one two") + + self.assertEqual(result.cmd, ['test', 'one', 'two']) + self.assertEqual(result.stdout, None) + self.assertEqual(result.stderr, None) + self.assertEqual(result.return_code, 0) + self.assertFalse(mock_canonicalize_path.called) + mock_backoff.assert_called_once_with(1) + mock_Popen.assert_called_once_with( + ['test', 'one', 'two'], env=env, cwd='/some/path', close_fds=True) + logger.assert_has_calls([ + mock.call.debug( + "Notice: splitting command string 'test one two'"), + mock.call.debug( + "Executing command: ['test', 'one', 'two'] (cwd /some/path)"), + ]) + self.assertEqual(len(logger.method_calls), 2) + + @mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2') + @mock.patch.object(os, 'getcwd', return_value='/some/path') + @mock.patch.object(environment.Environment, 'chdir') + @mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path') + @mock.patch.object(utils, 'backoff', return_value=[0]) + @mock.patch.object(subprocess, 'Popen', return_value=mock.Mock(**{ + 'returncode': 0, + 'communicate.return_value': (None, None), + })) + def test_call_cwd(self, mock_Popen, mock_backoff, mock_canonicalize_path, + mock_chdir, mock_getcwd): + logger = mock.Mock() + env = environment.Environment(logger) + + result = env(['test', 'one', 'two'], cwd='/other/path') + + self.assertEqual(result.cmd, ['test', 'one', 'two']) + self.assertEqual(result.stdout, None) + self.assertEqual(result.stderr, None) + self.assertEqual(result.return_code, 0) + mock_canonicalize_path.assert_called_once_with( + '/some/path', '/other/path') + mock_backoff.assert_called_once_with(1) + mock_Popen.assert_called_once_with( + ['test', 'one', 'two'], env=env, cwd='/canon/path', close_fds=True) + logger.assert_has_calls([ + mock.call.debug( + "Executing command: ['test', 'one', 'two'] (cwd /canon/path)"), + ]) + self.assertEqual(len(logger.method_calls), 1) + + @mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2') + @mock.patch.object(os, 'getcwd', return_value='/some/path') + @mock.patch.object(environment.Environment, 'chdir') + @mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path') + @mock.patch.object(utils, 'backoff', return_value=[0]) + @mock.patch.object(subprocess, 'Popen', return_value=mock.Mock(**{ + 'returncode': 0, + 'communicate.return_value': ('output', 'error'), + })) + def test_call_capture(self, mock_Popen, mock_backoff, + mock_canonicalize_path, mock_chdir, mock_getcwd): + logger = mock.Mock() + env = environment.Environment(logger) + + result = env(['test', 'one', 'two'], capture_output=True) + + self.assertEqual(result.cmd, ['test', 'one', 'two']) + self.assertEqual(result.stdout, 'output') + self.assertEqual(result.stderr, 'error') + self.assertEqual(result.return_code, 0) + self.assertFalse(mock_canonicalize_path.called) + mock_backoff.assert_called_once_with(1) + mock_Popen.assert_called_once_with( + ['test', 'one', 'two'], env=env, cwd='/some/path', close_fds=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + logger.assert_has_calls([ + mock.call.debug( + "Executing command: ['test', 'one', 'two'] (cwd /some/path)"), + ]) + self.assertEqual(len(logger.method_calls), 1) + + @mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2') + @mock.patch.object(os, 'getcwd', return_value='/some/path') + @mock.patch.object(environment.Environment, 'chdir') + @mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path') + @mock.patch.object(utils, 'backoff', return_value=[0]) + @mock.patch.object(subprocess, 'Popen', return_value=mock.Mock(**{ + 'returncode': 1, + 'communicate.return_value': (None, None), + })) + def test_call_failure_raise(self, mock_Popen, mock_backoff, + mock_canonicalize_path, mock_chdir, + mock_getcwd): + logger = mock.Mock() + env = environment.Environment(logger) + + try: + result = env(['test', 'one', 'two']) + except environment.ExecResult as exc: + self.assertEqual(exc.cmd, ['test', 'one', 'two']) + self.assertEqual(exc.stdout, None) + self.assertEqual(exc.stderr, None) + self.assertEqual(exc.return_code, 1) + else: + self.fail("Expected ExecResult to be raised") + + self.assertFalse(mock_canonicalize_path.called) + mock_backoff.assert_called_once_with(1) + mock_Popen.assert_called_once_with( + ['test', 'one', 'two'], env=env, cwd='/some/path', close_fds=True) + logger.assert_has_calls([ + mock.call.debug( + "Executing command: ['test', 'one', 'two'] (cwd /some/path)"), + ]) + self.assertEqual(len(logger.method_calls), 1) + + @mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2') + @mock.patch.object(os, 'getcwd', return_value='/some/path') + @mock.patch.object(environment.Environment, 'chdir') + @mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path') + @mock.patch.object(utils, 'backoff', return_value=[0]) + @mock.patch.object(subprocess, 'Popen', return_value=mock.Mock(**{ + 'returncode': 1, + 'communicate.return_value': (None, None), + })) + def test_call_failure_noraise(self, mock_Popen, mock_backoff, + mock_canonicalize_path, mock_chdir, + mock_getcwd): + logger = mock.Mock() + env = environment.Environment(logger) + + result = env(['test', 'one', 'two'], do_raise=False) + + self.assertEqual(result.cmd, ['test', 'one', 'two']) + self.assertEqual(result.stdout, None) + self.assertEqual(result.stderr, None) + self.assertEqual(result.return_code, 1) + self.assertFalse(mock_canonicalize_path.called) + mock_backoff.assert_called_once_with(1) + mock_Popen.assert_called_once_with( + ['test', 'one', 'two'], env=env, cwd='/some/path', close_fds=True) + logger.assert_has_calls([ + mock.call.debug( + "Executing command: ['test', 'one', 'two'] (cwd /some/path)"), + ]) + self.assertEqual(len(logger.method_calls), 1) + + @mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2') + @mock.patch.object(os, 'getcwd', return_value='/some/path') + @mock.patch.object(environment.Environment, 'chdir') + @mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path') + @mock.patch.object(utils, 'backoff', return_value=[0]) + @mock.patch.object(subprocess, 'Popen', return_value=mock.Mock(**{ + 'returncode': 0, + 'communicate.return_value': ('output', 'error'), + })) + def test_call_retry_success(self, mock_Popen, mock_backoff, + mock_canonicalize_path, mock_chdir, + mock_getcwd): + logger = mock.Mock() + env = environment.Environment(logger) + retry = mock.Mock(return_value=True) + + result = env(['test', 'one', 'two'], retry=retry) + + self.assertEqual(result.cmd, ['test', 'one', 'two']) + self.assertEqual(result.stdout, 'output') + self.assertEqual(result.stderr, 'error') + self.assertEqual(result.return_code, 0) + self.assertFalse(mock_canonicalize_path.called) + mock_backoff.assert_called_once_with(5) + mock_Popen.assert_called_once_with( + ['test', 'one', 'two'], env=env, cwd='/some/path', close_fds=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + logger.assert_has_calls([ + mock.call.debug( + "Executing command: ['test', 'one', 'two'] (cwd /some/path)"), + ]) + self.assertEqual(len(logger.method_calls), 1) + self.assertFalse(retry.called) + + @mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2') + @mock.patch.object(os, 'getcwd', return_value='/some/path') + @mock.patch.object(environment.Environment, 'chdir') + @mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path') + @mock.patch.object(utils, 'backoff', return_value=[0]) + @mock.patch.object(subprocess, 'Popen', return_value=mock.Mock(**{ + 'returncode': 0, + 'communicate.return_value': (None, None), + })) + def test_call_retry_success_badretries(self, mock_Popen, mock_backoff, + mock_canonicalize_path, mock_chdir, + mock_getcwd): + logger = mock.Mock() + env = environment.Environment(logger) + retry = mock.Mock(return_value=True) + + result = env(['test', 'one', 'two'], retry=retry, max_tries=-1) + + self.assertEqual(result.cmd, ['test', 'one', 'two']) + self.assertEqual(result.stdout, None) + self.assertEqual(result.stderr, None) + self.assertEqual(result.return_code, 0) + self.assertFalse(mock_canonicalize_path.called) + mock_backoff.assert_called_once_with(1) + mock_Popen.assert_called_once_with( + ['test', 'one', 'two'], env=env, cwd='/some/path', close_fds=True) + logger.assert_has_calls([ + mock.call.debug( + "Executing command: ['test', 'one', 'two'] (cwd /some/path)"), + ]) + self.assertEqual(len(logger.method_calls), 1) + self.assertFalse(retry.called) + + @mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2') + @mock.patch.object(os, 'getcwd', return_value='/some/path') + @mock.patch.object(environment.Environment, 'chdir') + @mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path') + @mock.patch.object(utils, 'backoff', return_value=[0, 1, 2, 3, 4, 5, 6]) + @mock.patch.object(subprocess, 'Popen', return_value=mock.Mock(**{ + 'returncode': 0, + 'communicate.return_value': ('output', 'error'), + })) + def test_call_retry_withtries(self, mock_Popen, mock_backoff, + mock_canonicalize_path, mock_chdir, + mock_getcwd): + logger = mock.Mock() + env = environment.Environment(logger) + retry = mock.Mock(return_value=True) + exec_results = [ + mock.Mock(__nonzero__=mock.Mock(return_value=False), + __bool__=mock.Mock(return_value=False)), + mock.Mock(__nonzero__=mock.Mock(return_value=False), + __bool__=mock.Mock(return_value=False)), + mock.Mock(__nonzero__=mock.Mock(return_value=True), + __bool__=mock.Mock(return_value=True)), + ] + + with mock.patch.object(environment, 'ExecResult', + side_effect=exec_results) as mock_ExecResult: + result = env(['test', 'one', 'two'], retry=retry, max_tries=7) + + self.assertEqual(result, exec_results[-1]) + self.assertFalse(mock_canonicalize_path.called) + mock_backoff.assert_called_once_with(7) + mock_Popen.assert_has_calls([ + mock.call(['test', 'one', 'two'], env=env, cwd='/some/path', + close_fds=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE), + mock.call(['test', 'one', 'two'], env=env, cwd='/some/path', + close_fds=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE), + mock.call(['test', 'one', 'two'], env=env, cwd='/some/path', + close_fds=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE), + ]) + self.assertEqual(mock_Popen.call_count, 3) + mock_ExecResult.assert_has_calls([ + mock.call(['test', 'one', 'two'], 'output', 'error', 0), + mock.call(['test', 'one', 'two'], 'output', 'error', 0), + mock.call(['test', 'one', 'two'], 'output', 'error', 0), + ]) + self.assertEqual(mock_ExecResult.call_count, 3) + logger.assert_has_calls([ + mock.call.debug( + "Executing command: ['test', 'one', 'two'] (cwd /some/path)"), + mock.call.warn('Failure caught; retrying command (try #2)'), + mock.call.warn('Failure caught; retrying command (try #3)'), + ]) + self.assertEqual(len(logger.method_calls), 3) + retry.assert_has_calls([mock.call(res) for res in exec_results[:-1]]) + self.assertEqual(retry.call_count, len(exec_results) - 1) + + @mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2') + @mock.patch.object(os, 'getcwd', return_value='/some/path') + @mock.patch.object(environment.Environment, 'chdir') + @mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path') + @mock.patch.object(utils, 'backoff', return_value=[0, 1]) + @mock.patch.object(subprocess, 'Popen', return_value=mock.Mock(**{ + 'returncode': 0, + 'communicate.return_value': ('output', 'error'), + })) + def test_call_retry_withtries_failure(self, mock_Popen, mock_backoff, + mock_canonicalize_path, mock_chdir, + mock_getcwd): + logger = mock.Mock() + env = environment.Environment(logger) + retry = mock.Mock(return_value=True) + exec_results = [ + mock.Mock(__nonzero__=mock.Mock(return_value=False), + __bool__=mock.Mock(return_value=False)), + mock.Mock(__nonzero__=mock.Mock(return_value=False), + __bool__=mock.Mock(return_value=False)), + mock.Mock(__nonzero__=mock.Mock(return_value=True), + __bool__=mock.Mock(return_value=True)), + ] + + with mock.patch.object(environment, 'ExecResult', + side_effect=exec_results) as mock_ExecResult: + result = env(['test', 'one', 'two'], retry=retry, max_tries=2, + do_raise=False) + + self.assertEqual(result, exec_results[-2]) + self.assertFalse(mock_canonicalize_path.called) + mock_backoff.assert_called_once_with(2) + mock_Popen.assert_has_calls([ + mock.call(['test', 'one', 'two'], env=env, cwd='/some/path', + close_fds=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE), + mock.call(['test', 'one', 'two'], env=env, cwd='/some/path', + close_fds=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE), + ]) + self.assertEqual(mock_Popen.call_count, 2) + mock_ExecResult.assert_has_calls([ + mock.call(['test', 'one', 'two'], 'output', 'error', 0), + mock.call(['test', 'one', 'two'], 'output', 'error', 0), + ]) + self.assertEqual(mock_ExecResult.call_count, 2) + logger.assert_has_calls([ + mock.call.debug( + "Executing command: ['test', 'one', 'two'] (cwd /some/path)"), + mock.call.warn('Failure caught; retrying command (try #2)'), + mock.call.warn('Unable to retry: too many attempts'), + ]) + self.assertEqual(len(logger.method_calls), 3) + retry.assert_has_calls([mock.call(res) for res in exec_results[:-2]]) + self.assertEqual(retry.call_count, len(exec_results) - 1) + + @mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2') + @mock.patch.object(os, 'getcwd', return_value='/some/path') + @mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path') + def test_chdir(self, mock_canonicalize_path, mock_getcwd): + with mock.patch.object(environment.Environment, 'chdir'): + env = environment.Environment('logger') + + result = env.chdir('test/directory') + + self.assertEqual(result, '/canon/path') + self.assertEqual(env.cwd, '/canon/path') + mock_canonicalize_path.assert_called_once_with( + '/some/path', 'test/directory') + + @mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2', + PATH='/bin') + @mock.patch.object(os, 'getcwd', return_value='/some/path') + @mock.patch.object(os.path, 'exists', return_value=False) + @mock.patch('shutil.rmtree') + @mock.patch.object(environment.Environment, 'chdir') + @mock.patch.object(environment.Environment, '__call__') + @mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path') + def test_create_venv_basic(self, mock_canonicalize_path, mock_call, + mock_chdir, mock_rmtree, mock_exists, + mock_getcwd): + logger = mock.Mock() + env = environment.Environment(logger) + expected = dict(env) + expected.update({ + 'VIRTUAL_ENV': '/canon/path', + 'PATH': '/canon/path/bin:/bin', + }) + mock_chdir.reset_mock() + + new_env = env.create_venv('venv/dir') + + self.assertNotEqual(id(new_env), id(env)) + self.assertTrue(isinstance(new_env, environment.Environment)) + self.assertEqual(new_env, expected) + self.assertEqual(new_env.logger, logger) + self.assertEqual(new_env.cwd, '/some/path') + self.assertEqual(new_env.venv_home, '/canon/path') + mock_canonicalize_path.assert_called_once_with( + '/some/path', 'venv/dir') + mock_exists.assert_called_once_with('/canon/path') + self.assertFalse(mock_rmtree.called) + mock_call.assert_called_once_with(['virtualenv', '/canon/path']) + mock_chdir.assert_called_once_with('/canon/path') + logger.assert_has_calls([ + mock.call.debug('Preparing virtual environment /canon/path'), + mock.call.info('Creating virtual environment /canon/path'), + ]) + self.assertEqual(len(logger.method_calls), 2) + + @mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2', + PATH='/bin') + @mock.patch.object(os, 'getcwd', return_value='/some/path') + @mock.patch.object(os.path, 'exists', return_value=False) + @mock.patch('shutil.rmtree') + @mock.patch.object(environment.Environment, 'chdir') + @mock.patch.object(environment.Environment, '__call__') + @mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path') + def test_create_venv_update(self, mock_canonicalize_path, mock_call, + mock_chdir, mock_rmtree, mock_exists, + mock_getcwd): + logger = mock.Mock() + env = environment.Environment(logger) + expected = dict(env) + expected.update({ + 'VIRTUAL_ENV': 'bah', + 'PATH': '/canon/path/bin:/bin', + 'a': 'foo', + }) + mock_chdir.reset_mock() + + new_env = env.create_venv('venv/dir', VIRTUAL_ENV='bah', a='foo') + + self.assertNotEqual(id(new_env), id(env)) + self.assertTrue(isinstance(new_env, environment.Environment)) + self.assertEqual(new_env, expected) + self.assertEqual(new_env.logger, logger) + self.assertEqual(new_env.cwd, '/some/path') + self.assertEqual(new_env.venv_home, '/canon/path') + mock_canonicalize_path.assert_called_once_with( + '/some/path', 'venv/dir') + mock_exists.assert_called_once_with('/canon/path') + self.assertFalse(mock_rmtree.called) + mock_call.assert_called_once_with(['virtualenv', '/canon/path']) + mock_chdir.assert_called_once_with('/canon/path') + logger.assert_has_calls([ + mock.call.debug('Preparing virtual environment /canon/path'), + mock.call.info('Creating virtual environment /canon/path'), + ]) + self.assertEqual(len(logger.method_calls), 2) + + @mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2', + PATH='/bin') + @mock.patch.object(os, 'getcwd', return_value='/some/path') + @mock.patch.object(os.path, 'exists', return_value=True) + @mock.patch('shutil.rmtree') + @mock.patch.object(environment.Environment, 'chdir') + @mock.patch.object(environment.Environment, '__call__') + @mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path') + def test_create_venv_exists(self, mock_canonicalize_path, mock_call, + mock_chdir, mock_rmtree, mock_exists, + mock_getcwd): + logger = mock.Mock() + env = environment.Environment(logger) + expected = dict(env) + expected.update({ + 'VIRTUAL_ENV': '/canon/path', + 'PATH': '/canon/path/bin:/bin', + }) + mock_chdir.reset_mock() + + new_env = env.create_venv('venv/dir') + + self.assertNotEqual(id(new_env), id(env)) + self.assertTrue(isinstance(new_env, environment.Environment)) + self.assertEqual(new_env, expected) + self.assertEqual(new_env.logger, logger) + self.assertEqual(new_env.cwd, '/some/path') + self.assertEqual(new_env.venv_home, '/canon/path') + mock_canonicalize_path.assert_called_once_with( + '/some/path', 'venv/dir') + mock_exists.assert_called_once_with('/canon/path') + self.assertFalse(mock_rmtree.called) + self.assertFalse(mock_call.called) + mock_chdir.assert_called_once_with('/canon/path') + logger.assert_has_calls([ + mock.call.debug('Preparing virtual environment /canon/path'), + mock.call.info('Using existing virtual environment /canon/path'), + ]) + self.assertEqual(len(logger.method_calls), 2) + + @mock.patch.dict(os.environ, clear=True, TEST_VAR1='1', TEST_VAR2='2', + PATH='/bin') + @mock.patch.object(os, 'getcwd', return_value='/some/path') + @mock.patch.object(os.path, 'exists', return_value=True) + @mock.patch('shutil.rmtree') + @mock.patch.object(environment.Environment, 'chdir') + @mock.patch.object(environment.Environment, '__call__') + @mock.patch.object(utils, 'canonicalize_path', return_value='/canon/path') + def test_create_venv_rebuild(self, mock_canonicalize_path, mock_call, + mock_chdir, mock_rmtree, mock_exists, + mock_getcwd): + logger = mock.Mock() + env = environment.Environment(logger) + expected = dict(env) + expected.update({ + 'VIRTUAL_ENV': '/canon/path', + 'PATH': '/canon/path/bin:/bin', + }) + mock_chdir.reset_mock() + + new_env = env.create_venv('venv/dir', True) + + self.assertNotEqual(id(new_env), id(env)) + self.assertTrue(isinstance(new_env, environment.Environment)) + self.assertEqual(new_env, expected) + self.assertEqual(new_env.logger, logger) + self.assertEqual(new_env.cwd, '/some/path') + self.assertEqual(new_env.venv_home, '/canon/path') + mock_canonicalize_path.assert_called_once_with( + '/some/path', 'venv/dir') + mock_exists.assert_called_once_with('/canon/path') + mock_rmtree.assert_called_once_with('/canon/path') + mock_call.assert_called_once_with(['virtualenv', '/canon/path']) + mock_chdir.assert_called_once_with('/canon/path') + logger.assert_has_calls([ + mock.call.debug('Preparing virtual environment /canon/path'), + mock.call.info('Destroying old virtual environment /canon/path'), + mock.call.info('Creating virtual environment /canon/path'), + ]) + self.assertEqual(len(logger.method_calls), 3)