diff --git a/TODO b/TODO index 151270d..bc3453d 100644 --- a/TODO +++ b/TODO @@ -3,3 +3,8 @@ fail at test_changeset.test_fk(..)) - better SQL scripts support (testing, source viewing) + +make_update_script_for_model: +- calculated differences between models are actually differences between metas +- columns are not compared? +- even if two "models" are equal, it doesn't yield so diff --git a/docs/api.rst b/docs/api.rst index 92d9031..6f0a6f2 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -135,6 +135,23 @@ Module :mod:`shell <migrate.versioning.shell>` :members: :synopsis: Shell commands +Module :mod:`script <migrate.versioning.script>` +------------------------------------------------ + +.. automodule:: migrate.versioning.script.base + :synopsis: Script utilities + :members: + +.. automodule:: migrate.versioning.script.py + :members: + :inherited-members: + :show-inheritance: + +.. automodule:: migrate.versioning.script.sql + :members: + :show-inheritance: + :inherited-members: + Module :mod:`util <migrate.versioning.util>` ------------------------------------------------ diff --git a/migrate/versioning/repository.py b/migrate/versioning/repository.py index 4e705a4..16f4b80 100644 --- a/migrate/versioning/repository.py +++ b/migrate/versioning/repository.py @@ -61,6 +61,7 @@ class Changeset(dict): class Repository(pathed.Pathed): """A project's change script repository""" + _config = 'migrate.cfg' _versions = 'versions' @@ -68,8 +69,8 @@ class Repository(pathed.Pathed): log.info('Loading repository %s...' % path) self.verify(path) super(Repository, self).__init__(path) - self.config=cfgparse.Config(os.path.join(self.path, self._config)) - self.versions=version.Collection(os.path.join(self.path, + self.config = cfgparse.Config(os.path.join(self.path, self._config)) + self.versions = version.Collection(os.path.join(self.path, self._versions)) log.info('Repository %s loaded successfully' % path) log.debug('Config: %r' % self.config.to_dict()) @@ -116,6 +117,7 @@ class Repository(pathed.Pathed): tmplpkg = '.'.join((pkg, rsrc)) tmplfile = resource_filename(pkg, rsrc) config_text = cls.prepare_config(tmplpkg, cls._config, name, **opts) + # Create repository try: shutil.copytree(tmplfile, path) @@ -136,10 +138,17 @@ class Repository(pathed.Pathed): def create_script_sql(self, database, **k): self.versions.create_new_sql_version(database, **k) - latest=property(lambda self: self.versions.latest) - version_table=property(lambda self: self.config.get('db_settings', - 'version_table')) - id=property(lambda self: self.config.get('db_settings', 'repository_id')) + @property + def latest(self): + return self.versions.latest + + @property + def version_table(self): + return self.config.get('db_settings', 'version_table') + + @property + def id(self): + return self.config.get('db_settings', 'repository_id') def version(self, *p, **k): return self.versions.version(*p, **k) @@ -177,7 +186,7 @@ def manage(file, **opts): pkg, rsrc = template.manage(as_pkg=True) tmpl = resource_string(pkg, rsrc) vars = ",".join(["%s='%s'" % vars for vars in opts.iteritems()]) - result = tmpl%dict(defaults=vars) + result = tmpl % dict(defaults=vars) fd = open(file, 'w') fd.write(result) diff --git a/migrate/versioning/script/__init__.py b/migrate/versioning/script/__init__.py index ac8d1f4..c788eda 100644 --- a/migrate/versioning/script/__init__.py +++ b/migrate/versioning/script/__init__.py @@ -1,3 +1,6 @@ -from py import PythonScript -from sql import SqlScript -from base import BaseScript +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from migrate.versioning.script.base import BaseScript +from migrate.versioning.script.py import PythonScript +from migrate.versioning.script.sql import SqlScript diff --git a/migrate/versioning/script/base.py b/migrate/versioning/script/base.py index 55aadd3..2fdc5df 100644 --- a/migrate/versioning/script/base.py +++ b/migrate/versioning/script/base.py @@ -1,12 +1,12 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from migrate.versioning.base import log,operations -from migrate.versioning import pathed,exceptions +from migrate.versioning.base import log, operations +from migrate.versioning import pathed, exceptions class BaseScript(pathed.Pathed): - """Base class for other types of scripts + """Base class for other types of scripts. All scripts have the following properties: source (script.source()) @@ -17,18 +17,20 @@ class BaseScript(pathed.Pathed): The operations defined by the script: upgrade(), downgrade() or both. Returns a tuple of operations. Can also check for an operation with ex. script.operation(Script.ops.up) - """ + """ # TODO: sphinxfy this and implement it correctly - def __init__(self,path): + def __init__(self, path): log.info('Loading script %s...' % path) self.verify(path) super(BaseScript, self).__init__(path) log.info('Script %s loaded successfully' % path) @classmethod - def verify(cls,path): - """Ensure this is a valid script, or raise InvalidScriptError + def verify(cls, path): + """Ensure this is a valid script This version simply ensures the script file's existence + + :raises: :exc:`InvalidScriptError <migrate.versioning.exceptions.InvalidScriptError>` """ try: cls.require_found(path) @@ -36,10 +38,16 @@ class BaseScript(pathed.Pathed): raise exceptions.InvalidScriptError(path) def source(self): + """:returns: source code of the script. + :rtype: string + """ fd = open(self.path) ret = fd.read() fd.close() return ret def run(self, engine): + """Core of each BaseScript subclass. + This method executes the script. + """ raise NotImplementedError() diff --git a/migrate/versioning/script/py.py b/migrate/versioning/script/py.py index cd7c634..e942f26 100644 --- a/migrate/versioning/script/py.py +++ b/migrate/versioning/script/py.py @@ -12,10 +12,13 @@ from migrate.versioning.script import base from migrate.versioning.util import import_path, load_model, construct_engine class PythonScript(base.BaseScript): + """Base for Python scripts""" @classmethod def create(cls, path, **opts): - """Create an empty migration script""" + """Create an empty migration script at specified path + + :returns: :class:`PythonScript instance <migrate.versioning.script.py.PythonScript>`""" cls.require_notfound(path) # TODO: Use the default script template (defined in the template @@ -25,30 +28,51 @@ class PythonScript(base.BaseScript): src = template.get_script(template_file) shutil.copy(src, path) + return cls(path) + @classmethod def make_update_script_for_model(cls, engine, oldmodel, model, repository, **opts): - """Create a migration script""" + """Create a migration script based on difference between two SA models. + + :param repository: path to migrate repository + :param oldmodel: dotted.module.name:SAClass or SAClass object + :param model: dotted.module.name:SAClass or SAClass object + :param engine: SQLAlchemy engine + :type repository: string or :class:`Repository instance <migrate.versioning.repository.Repository>` + :type oldmodel: string or Class + :type model: string or Class + :type engine: Engine instance + :returns: Upgrade / Downgrade script + :rtype: string + """ - # Compute differences. if isinstance(repository, basestring): # oh dear, an import cycle! from migrate.versioning.repository import Repository repository = Repository(repository) + oldmodel = load_model(oldmodel) model = load_model(model) + + # Compute differences. diff = schemadiff.getDiffOfModelAgainstModel( oldmodel, model, engine, excludeTables=[repository.version_table]) + # TODO: diff can be False (there is no difference?) decls, upgradeCommands, downgradeCommands = \ genmodel.ModelGenerator(diff).toUpgradeDowngradePython() # Store differences into file. - template_file = None - src = template.get_script(template_file) - contents = open(src).read() + # TODO: add custom templates + src = template.get_script(None) + f = open(src) + contents = f.read() + f.close() + + # generate source search = 'def upgrade():' contents = contents.replace(search, '\n\n'.join((decls, search)), 1) if upgradeCommands: @@ -58,11 +82,18 @@ class PythonScript(base.BaseScript): return contents @classmethod - def verify_module(cls,path): - """Ensure this is a valid script, or raise InvalidScriptError""" + def verify_module(cls, path): + """Ensure path is a valid script + + :param path: Script location + :type path: string + + :raises: :exc:`InvalidScriptError <migrate.versioning.exceptions.InvalidScriptError>` + :returns: Python module + """ # Try to import and get the upgrade() func try: - module=import_path(path) + module = import_path(path) except: # If the script itself has errors, that's not our problem raise @@ -73,8 +104,11 @@ class PythonScript(base.BaseScript): return module def preview_sql(self, url, step, **args): - """Mock engine to store all executable calls in a string \ - and execute the step""" + """Mocks SQLAlchemy Engine to store all executed calls in a string + and runs :meth:`PythonScript.run <migrate.versioning.script.py.PythonScript.run>` + + :returns: SQL file + """ buf = StringIO() args['engine_arg_strategy'] = 'mock' args['engine_arg_executor'] = lambda s, p='': buf.write(s + p) @@ -85,8 +119,14 @@ class PythonScript(base.BaseScript): return buf.getvalue() def run(self, engine, step): - """Core method of Script file. \ - Exectues update() or downgrade() function""" + """Core method of Script file. + Exectues :func:`update` or :func:`downgrade` functions + + :param engine: SQLAlchemy Engine + :param step: Operation to run + :type engine: string + :type step: int + """ if step > 0: op = 'upgrade' elif step < 0: @@ -104,13 +144,16 @@ class PythonScript(base.BaseScript): @property def module(self): - if not hasattr(self,'_module'): + """Calls :meth:`migrate.versioning.script.py.verify_module` + and returns it. + """ + if not hasattr(self, '_module'): self._module = self.verify_module(self.path) return self._module def _func(self, funcname): - fn = getattr(self.module, funcname, None) - if not fn: + try: + return getattr(self.module, funcname) + except AttributeError: msg = "The function %s is not defined in this script" - raise exceptions.ScriptError(msg%funcname) - return fn + raise exceptions.ScriptError(msg % funcname) diff --git a/migrate/versioning/script/sql.py b/migrate/versioning/script/sql.py index 88c3bf8..851fdf2 100644 --- a/migrate/versioning/script/sql.py +++ b/migrate/versioning/script/sql.py @@ -1,8 +1,15 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + from migrate.versioning.script import base + class SqlScript(base.BaseScript): """A file containing plain SQL statements.""" - def run(self, engine, step): + + # TODO: why is step parameter even here? + def run(self, engine, step=None): + """Runs SQL script through raw dbapi execute call""" text = self.source() # Don't rely on SA's autocommit here # (SA uses .startswith to check if a commit is needed. What if script @@ -11,14 +18,13 @@ class SqlScript(base.BaseScript): try: trans = conn.begin() try: - # ###HACK: SQLite doesn't allow multiple statements through + # HACK: SQLite doesn't allow multiple statements through # its execute() method, but it provides executescript() instead dbapi = conn.engine.raw_connection() if getattr(dbapi, 'executescript', None): dbapi.executescript(text) else: conn.execute(text) - # Success trans.commit() except: trans.rollback() diff --git a/migrate/versioning/util/__init__.py b/migrate/versioning/util/__init__.py index e0060ad..6e41067 100644 --- a/migrate/versioning/util/__init__.py +++ b/migrate/versioning/util/__init__.py @@ -91,8 +91,13 @@ def construct_engine(url, **opts): Currently, there are 2 ways to pass create_engine options to :mod:`migrate.versioning.api` functions: + :param url: connection string :param engine_dict: python dictionary of options to pass to `create_engine` :param engine_arg_*: keyword parameters to pass to `create_engine` (evaluated with :func:`migrate.versioning.util.guess_obj_type`) + :type engine_dict: dict + :type url: string + :type engine_arg_*: string + :returns: SQLAlchemy Engine .. note:: diff --git a/setup.cfg b/setup.cfg index 332464c..dd70583 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,8 +7,8 @@ tag_svn_revision = 1 tag_build = .dev [nosetests] -#pdb = true -#pdb-failures = true +pdb = true +pdb-failures = true [aliases] release = egg_info -RDb '' diff --git a/test/fixture/pathed.py b/test/fixture/pathed.py index 728a613..e8c6ebc 100644 --- a/test/fixture/pathed.py +++ b/test/fixture/pathed.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import os +import sys import shutil import tempfile @@ -16,10 +17,12 @@ class Pathed(base.Base): def setUp(self): super(Pathed, self).setUp() self.temp_usable_dir = tempfile.mkdtemp() + sys.path.append(self.temp_usable_dir) def tearDown(self): super(Pathed, self).tearDown() - self.temp_usable_dir = tempfile.mkdtemp() + sys.path.remove(self.temp_usable_dir) + Pathed.purge(self.temp_usable_dir) @classmethod def _tmp(cls, prefix='', suffix=''): diff --git a/test/versioning/test_script.py b/test/versioning/test_script.py index 4715b99..ec162d4 100644 --- a/test/versioning/test_script.py +++ b/test/versioning/test_script.py @@ -2,13 +2,29 @@ # -*- coding: utf-8 -*- import os +import sys import shutil +from migrate.versioning import exceptions, version, repository from migrate.versioning.script import * -from migrate.versioning import exceptions, version +from migrate.versioning.util import * + from test import fixture +class TestBaseScript(fixture.Pathed): + + def test_all(self): + """Testing all basic BaseScript operations""" + # verify / source / run + src = self.tmp() + open(src, 'w').close() + bscript = BaseScript(src) + BaseScript.verify(src) + self.assertEqual(bscript.source(), '') + self.assertRaises(NotImplementedError, bscript.run, 'foobar') + + class TestPyScript(fixture.Pathed, fixture.DB): cls = PythonScript def test_create(self): @@ -22,6 +38,16 @@ class TestPyScript(fixture.Pathed, fixture.DB): # Can't create it again: it already exists self.assertRaises(exceptions.PathFoundError,self.cls.create,path) + @fixture.usedb(supported='sqlite') + def test_run(self): + script_path = self.tmp_py() + pyscript = PythonScript.create(script_path) + pyscript.run(self.engine, 1) + pyscript.run(self.engine, -1) + + self.assertRaises(exceptions.ScriptError, pyscript.run, self.engine, 0) + self.assertRaises(exceptions.ScriptError, pyscript._func, 'foobar') + def test_verify_notfound(self): """Correctly verify a python migration script: nonexistant file""" path = self.tmp_py() @@ -93,7 +119,80 @@ def upgrade(): # Succeeds after creating self.cls.create(path) self.cls.verify(path) - -class TestSqlScript(fixture.Pathed): - pass - + + # test for PythonScript.make_update_script_for_model + + @fixture.usedb() + def test_make_update_script_for_model(self): + """Construct script source from differences of two models""" + + self.setup_model_params() + self.write_file(self.first_model_path, self.base_source) + self.write_file(self.second_model_path, self.base_source + self.model_source) + + + source_script = self.pyscript.make_update_script_for_model( + engine=self.engine, + oldmodel=load_model('testmodel_first:meta'), + model=load_model('testmodel_second:meta'), + repository=self.repo_path, + ) + + self.assertTrue('User.create()' in source_script) + self.assertTrue('User.drop()' in source_script) + + #@fixture.usedb() + #def test_make_update_script_for_model_equals(self): + # """Try to make update script from two identical models""" + + # self.setup_model_params() + # self.write_file(self.first_model_path, self.base_source + self.model_source) + # self.write_file(self.second_model_path, self.base_source + self.model_source) + + # source_script = self.pyscript.make_update_script_for_model( + # engine=self.engine, + # oldmodel=load_model('testmodel_first:meta'), + # model=load_model('testmodel_second:meta'), + # repository=self.repo_path, + # ) + + # self.assertFalse('User.create()' in source_script) + # self.assertFalse('User.drop()' in source_script) + + def setup_model_params(self): + self.script_path = self.tmp_py() + self.repo_path = self.tmp() + self.first_model_path = os.path.join(self.temp_usable_dir, 'testmodel_first.py') + self.second_model_path = os.path.join(self.temp_usable_dir, 'testmodel_second.py') + + self.base_source = """from sqlalchemy import *\nmeta = MetaData()\n""" + self.model_source = """ +User = Table('User', meta, + Column('id', Integer, primary_key=True), + Column('login', Unicode(40)), + Column('passwd', String(40)), +)""" + + self.repo = repository.Repository.create(self.repo_path, 'repo') + self.pyscript = PythonScript.create(self.script_path) + + + def write_file(self, path, contents): + f = open(path, 'w') + f.write(contents) + f.close() + + +class TestSqlScript(fixture.Pathed, fixture.DB): + + @fixture.usedb() + def test_error(self): + """Test if exception is raised on wrong script source""" + src = self.tmp() + + f = open(src, 'w') + f.write("""foobar""") + f.close() + + sqls = SqlScript(src) + self.assertRaises(Exception, sqls.run, self.engine) diff --git a/test/versioning/test_util.py b/test/versioning/test_util.py index 738bf37..e131fe6 100644 --- a/test/versioning/test_util.py +++ b/test/versioning/test_util.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import os + from test import fixture from migrate.versioning.util import * @@ -51,16 +53,16 @@ class TestUtil(fixture.Pathed): def test_load_model(self): """load model from dotted name""" - model_path = self.tmp_named('testmodel.py') + model_path = os.path.join(self.temp_usable_dir, 'test_load_model.py') f = open(model_path, 'w') f.write("class FakeFloat(int): pass") f.close() - FakeFloat = load_model('testmodel.FakeFloat') + FakeFloat = load_model('test_load_model.FakeFloat') self.assert_(isinstance(FakeFloat(), int)) - FakeFloat = load_model('testmodel:FakeFloat') + FakeFloat = load_model('test_load_model:FakeFloat') self.assert_(isinstance(FakeFloat(), int)) FakeFloat = load_model(FakeFloat)