Commit a03b141a authored by Cyril Roelandt's avatar Cyril Roelandt

Port to Python3

Brief summary of the modifications:

* Use six for compatibility with both Python 2 and 3;
* Replace UserDict.DictMixin with collections.MutableMapping;
* Fix relative imports;
* Use test-requirements.txt for requirements that are common to both Python 2
  and 3, and test-requirements-py{2,3}.txt for version-specific requirements;
* Miscellaneous fixes.
* Use a specific test_db_py3.cfg file for Python 3, that only runs tests on
  sqlite.

Thanks to Victor Stinner who co-wrote this patch.

Change-Id: Ia6dc536c39d274924c21fd5bb619e8e5721e04c4
Co-Authored-By: 's avatarVictor Stinner <victor.stinner@enovance.com>
parent 07909159
......@@ -4,7 +4,6 @@
At the moment, this isn't so much based off of ANSI as much as
things that just happen to work with multiple databases.
"""
import StringIO
import sqlalchemy as sa
from sqlalchemy.schema import SchemaVisitor
......@@ -20,6 +19,7 @@ from migrate import exceptions
import sqlalchemy.sql.compiler
from migrate.changeset import constraint
from migrate.changeset import util
from six.moves import StringIO
from sqlalchemy.schema import AddConstraint, DropConstraint
from sqlalchemy.sql.compiler import DDLCompiler
......@@ -43,11 +43,12 @@ class AlterTableVisitor(SchemaVisitor):
try:
return self.connection.execute(self.buffer.getvalue())
finally:
self.buffer.truncate(0)
self.buffer.seek(0)
self.buffer.truncate()
def __init__(self, dialect, connection, **kw):
self.connection = connection
self.buffer = StringIO.StringIO()
self.buffer = StringIO()
self.preparer = dialect.identifier_preparer
self.dialect = dialect
......
......@@ -3,7 +3,10 @@
.. _`SQLite`: http://www.sqlite.org/
"""
from UserDict import DictMixin
try: # Python 3
from collections import MutableMapping as DictMixin
except ImportError: # Python 2
from UserDict import DictMixin
from copy import copy
from sqlalchemy.databases import sqlite as sa_base
......
"""
Schema module providing common schema operations.
"""
import abc
try: # Python 3
from collections import MutableMapping as DictMixin
except ImportError: # Python 2
from UserDict import DictMixin
import warnings
from UserDict import DictMixin
import six
import sqlalchemy
from sqlalchemy.schema import ForeignKeyConstraint
......@@ -163,7 +167,39 @@ def _to_index(index, table=None, engine=None):
return ret
class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem):
# Python3: if we just use:
#
# class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem):
# ...
#
# We get the following error:
# TypeError: metaclass conflict: the metaclass of a derived class must be a
# (non-strict) subclass of the metaclasses of all its bases.
#
# The complete inheritance/metaclass relationship list of ColumnDelta can be
# summarized by this following dot file:
#
# digraph test123 {
# ColumnDelta -> MutableMapping;
# MutableMapping -> Mapping;
# Mapping -> {Sized Iterable Container};
# {Sized Iterable Container} -> ABCMeta[style=dashed];
#
# ColumnDelta -> SchemaItem;
# SchemaItem -> {SchemaEventTarget Visitable};
# SchemaEventTarget -> object;
# Visitable -> {VisitableType object} [style=dashed];
# VisitableType -> type;
# }
#
# We need to use a metaclass that inherits from all the metaclasses of
# DictMixin and sqlalchemy.schema.SchemaItem. Let's call it "MyMeta".
class MyMeta(sqlalchemy.sql.visitors.VisitableType, abc.ABCMeta, object):
pass
class ColumnDelta(six.with_metaclass(MyMeta, DictMixin, sqlalchemy.schema.SchemaItem)):
"""Extracts the differences between two columns/column-parameters
May receive parameters arranged in several different ways:
......@@ -229,7 +265,7 @@ class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem):
diffs = self.compare_1_column(*p, **kw)
else:
# Zero columns specified
if not len(p) or not isinstance(p[0], basestring):
if not len(p) or not isinstance(p[0], six.string_types):
raise ValueError("First argument must be column name")
diffs = self.compare_parameters(*p, **kw)
......@@ -254,6 +290,12 @@ class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem):
def __delitem__(self, key):
raise NotImplementedError
def __len__(self):
raise NotImplementedError
def __iter__(self):
raise NotImplementedError
def keys(self):
return self.diffs.keys()
......@@ -332,7 +374,7 @@ class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem):
"""Extracts data from p and modifies diffs"""
p = list(p)
while len(p):
if isinstance(p[0], basestring):
if isinstance(p[0], six.string_types):
k.setdefault('name', p.pop(0))
elif isinstance(p[0], sqlalchemy.types.TypeEngine):
k.setdefault('type', p.pop(0))
......@@ -370,7 +412,7 @@ class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem):
return getattr(self, '_table', None)
def _set_table(self, table):
if isinstance(table, basestring):
if isinstance(table, six.string_types):
if self.alter_metadata:
if not self.meta:
raise ValueError("metadata must be specified for table"
......@@ -587,7 +629,7 @@ populated with defaults
if isinstance(cons,(ForeignKeyConstraint,
UniqueConstraint)):
for col_name in cons.columns:
if not isinstance(col_name,basestring):
if not isinstance(col_name,six.string_types):
col_name = col_name.name
if self.name==col_name:
to_drop.add(cons)
......@@ -622,7 +664,7 @@ populated with defaults
if (getattr(self, name[:-5]) and not obj):
raise InvalidConstraintError("Column.create() accepts index_name,"
" primary_key_name and unique_name to generate constraints")
if not isinstance(obj, basestring) and obj is not None:
if not isinstance(obj, six.string_types) and obj is not None:
raise InvalidConstraintError(
"%s argument for column must be constraint name" % name)
......
......@@ -6,10 +6,11 @@ sys.path.append(os.path.dirname(os.path.abspath(__file__)))
from unittest import TestCase
import migrate
import six
class TestVersionDefined(TestCase):
def test_version(self):
"""Test for migrate.__version__"""
self.assertTrue(isinstance(migrate.__version__, basestring))
self.assertTrue(isinstance(migrate.__version__, six.string_types))
self.assertTrue(len(migrate.__version__) > 0)
......@@ -11,6 +11,7 @@ from migrate.changeset import constraint
from migrate.changeset.schema import ColumnDelta
from migrate.tests import fixture
from migrate.tests.fixture.warnings import catch_warnings
import six
class TestAddDropColumn(fixture.DB):
"""Test add/drop column through all possible interfaces
......@@ -400,7 +401,7 @@ class TestAddDropColumn(fixture.DB):
if isinstance(cons,ForeignKeyConstraint):
col_names = []
for col_name in cons.columns:
if not isinstance(col_name,basestring):
if not isinstance(col_name,six.string_types):
col_name = col_name.name
col_names.append(col_name)
result.append(col_names)
......@@ -612,7 +613,7 @@ class TestColumnChange(fixture.DB):
self.table.drop()
try:
self.table.create()
except sqlalchemy.exc.SQLError, e:
except sqlalchemy.exc.SQLError:
# SQLite: database schema has changed
if not self.url.startswith('sqlite://'):
raise
......@@ -621,7 +622,7 @@ class TestColumnChange(fixture.DB):
if self.table.exists():
try:
self.table.drop(self.engine)
except sqlalchemy.exc.SQLError,e:
except sqlalchemy.exc.SQLError:
# SQLite: database schema has changed
if not self.url.startswith('sqlite://'):
raise
......@@ -843,7 +844,7 @@ class TestColumnDelta(fixture.DB):
def verify(self, expected, original, *p, **k):
self.delta = ColumnDelta(original, *p, **k)
result = self.delta.keys()
result = list(self.delta.keys())
result.sort()
self.assertEqual(expected, result)
return self.delta
......
......@@ -12,7 +12,7 @@ def main(imports=None):
defaultTest=None
return testtools.TestProgram(defaultTest=defaultTest)
from base import Base
from migrate.tests.fixture.pathed import Pathed
from shell import Shell
from database import DB,usedb
from .base import Base
from .pathed import Pathed
from .shell import Shell
from .database import DB,usedb
......@@ -3,6 +3,9 @@
import os
import logging
import sys
import six
from decorator import decorator
from sqlalchemy import create_engine, Table, MetaData
......@@ -23,7 +26,7 @@ log = logging.getLogger(__name__)
def readurls():
"""read URLs from config file return a list"""
# TODO: remove tmpfile since sqlite can store db in memory
filename = 'test_db.cfg'
filename = 'test_db.cfg' if six.PY2 else "test_db_py3.cfg"
ret = list()
tmpfile = Pathed.tmp()
fullpath = os.path.join(os.curdir, filename)
......@@ -46,12 +49,12 @@ def is_supported(url, supported, not_supported):
db = url.split(':', 1)[0]
if supported is not None:
if isinstance(supported, basestring):
if isinstance(supported, six.string_types):
return supported == db
else:
return db in supported
elif not_supported is not None:
if isinstance(not_supported, basestring):
if isinstance(not_supported, six.string_types):
return not_supported != db
else:
return not (db in not_supported)
......@@ -96,7 +99,7 @@ def usedb(supported=None, not_supported=None):
finally:
try:
self._teardown()
except Exception,e:
except Exception as e:
teardown_exception=e
else:
teardown_exception=None
......@@ -106,14 +109,14 @@ def usedb(supported=None, not_supported=None):
'setup: %r\n'
'teardown: %r\n'
)%(setup_exception,teardown_exception))
except Exception,e:
except Exception:
failed_for.append(url)
fail = True
fail = sys.exc_info()
for url in failed_for:
log.error('Failed for %s', url)
if fail:
# cause the failure :-)
raise
six.reraise(*fail)
return dec
......
#!/usr/bin/python
# -*- coding: utf-8 -*-
import six
from migrate.exceptions import *
from migrate.versioning import api
......@@ -12,7 +14,7 @@ from migrate.tests import fixture
class TestAPI(Pathed):
def test_help(self):
self.assertTrue(isinstance(api.help('help'), basestring))
self.assertTrue(isinstance(api.help('help'), six.string_types))
self.assertRaises(UsageError, api.help)
self.assertRaises(UsageError, api.help, 'foobar')
self.assertTrue(isinstance(api.help('create'), str))
......@@ -48,7 +50,7 @@ class TestAPI(Pathed):
repo = self.tmp_repos()
api.create(repo, 'temp')
api.version_control('sqlite:///', repo)
api.version_control('sqlite:///', unicode(repo))
api.version_control('sqlite:///', six.text_type(repo))
def test_source(self):
repo = self.tmp_repos()
......
......@@ -2,6 +2,7 @@
import os
import six
import sqlalchemy
from sqlalchemy import *
......@@ -43,13 +44,12 @@ class TestSchemaDiff(fixture.DB):
# so the schema diffs on the columns don't work with this test.
@fixture.usedb(not_supported='ibm_db_sa')
def test_functional(self):
def assertDiff(isDiff, tablesMissingInDatabase, tablesMissingInModel, tablesWithDiff):
diff = schemadiff.getDiffOfModelAgainstDatabase(self.meta, self.engine, excludeTables=['migrate_version'])
self.assertEqual(
(diff.tables_missing_from_B,
diff.tables_missing_from_A,
diff.tables_different.keys(),
list(diff.tables_different.keys()),
bool(diff)),
(tablesMissingInDatabase,
tablesMissingInModel,
......@@ -97,10 +97,11 @@ class TestSchemaDiff(fixture.DB):
diff = schemadiff.getDiffOfModelAgainstDatabase(MetaData(), self.engine, excludeTables=['migrate_version'])
src = genmodel.ModelGenerator(diff,self.engine).genBDefinition()
exec src in locals()
namespace = {}
six.exec_(src, namespace)
c1 = Table('tmp_schemadiff', self.meta, autoload=True).c
c2 = tmp_schemadiff.c
c2 = namespace['tmp_schemadiff'].c
self.compare_columns_equal(c1, c2, ['type'])
# TODO: get rid of ignoring type
......@@ -139,19 +140,19 @@ class TestSchemaDiff(fixture.DB):
decls, upgradeCommands, downgradeCommands = genmodel.ModelGenerator(diff,self.engine).genB2AMigration(indent='')
# decls have changed since genBDefinition
exec decls in locals()
six.exec_(decls, namespace)
# migration commands expect a namespace containing migrate_engine
migrate_engine = self.engine
namespace['migrate_engine'] = self.engine
# run the migration up and down
exec upgradeCommands in locals()
six.exec_(upgradeCommands, namespace)
assertDiff(False, [], [], [])
exec decls in locals()
exec downgradeCommands in locals()
six.exec_(decls, namespace)
six.exec_(downgradeCommands, namespace)
assertDiff(True, [], [], [self.table_name])
exec decls in locals()
exec upgradeCommands in locals()
six.exec_(decls, namespace)
six.exec_(upgradeCommands, namespace)
assertDiff(False, [], [], [])
if not self.engine.name == 'oracle':
......
......@@ -111,7 +111,6 @@ class TestVersionedRepository(fixture.Pathed):
# Create a script and test again
now = int(datetime.utcnow().strftime('%Y%m%d%H%M%S'))
repos.create_script('')
print repos.latest
self.assertEqual(repos.latest, now)
def test_source(self):
......
......@@ -4,6 +4,8 @@
import os
import shutil
import six
from migrate import exceptions
from migrate.versioning.schema import *
from migrate.versioning import script, schemadiff
......@@ -163,10 +165,10 @@ class TestControlledSchema(fixture.Pathed, fixture.DB):
def test_create_model(self):
"""Test workflow to generate create_model"""
model = ControlledSchema.create_model(self.engine, self.repos, declarative=False)
self.assertTrue(isinstance(model, basestring))
self.assertTrue(isinstance(model, six.string_types))
model = ControlledSchema.create_model(self.engine, self.repos.path, declarative=True)
self.assertTrue(isinstance(model, basestring))
self.assertTrue(isinstance(model, six.string_types))
@fixture.usedb()
def test_compare_model_to_db(self):
......
......@@ -27,9 +27,9 @@ class SchemaDiffBase(fixture.DB):
# print diff
self.assertTrue(diff)
self.assertEqual(1,len(diff.tables_different))
td = diff.tables_different.values()[0]
td = list(diff.tables_different.values())[0]
self.assertEqual(1,len(td.columns_different))
cd = td.columns_different.values()[0]
cd = list(td.columns_different.values())[0]
label_width = max(len(self.name1), len(self.name2))
self.assertEqual(('Schema diffs:\n'
' table with differences: xtable\n'
......
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import imp
import os
import sys
import shutil
import six
from migrate import exceptions
from migrate.versioning import version, repository
from migrate.versioning.script import *
......@@ -51,7 +53,10 @@ class TestPyScript(fixture.Pathed, fixture.DB):
self.assertRaises(exceptions.ScriptError, pyscript._func, 'foobar')
# clean pyc file
os.remove(script_path + 'c')
if six.PY3:
os.remove(imp.cache_from_source(script_path))
else:
os.remove(script_path + 'c')
# test deprecated upgrade/downgrade with no arguments
contents = open(script_path, 'r').read()
......@@ -94,7 +99,7 @@ class TestPyScript(fixture.Pathed, fixture.DB):
path = self.tmp_py()
# Create empty file
f = open(path, 'w')
f.write("def zergling():\n\tprint 'rush'")
f.write("def zergling():\n\tprint('rush')")
f.close()
self.assertRaises(exceptions.InvalidScriptError, self.cls.verify_module, path)
# script isn't verified on creation, but on module reference
......
......@@ -5,7 +5,8 @@ import os
import sys
import tempfile
from cStringIO import StringIO
import six
from six.moves import cStringIO
from sqlalchemy import MetaData, Table
from migrate.exceptions import *
......@@ -29,7 +30,7 @@ class TestShellCommands(Shell):
# we can only test that we get some output
for cmd in api.__all__:
result = self.env.run('migrate help %s' % cmd)
self.assertTrue(isinstance(result.stdout, basestring))
self.assertTrue(isinstance(result.stdout, six.string_types))
self.assertTrue(result.stdout)
self.assertFalse(result.stderr)
......@@ -61,11 +62,11 @@ class TestShellCommands(Shell):
def _check_error(self,args,code,expected,**kw):
original = sys.stderr
try:
actual = StringIO()
actual = cStringIO()
sys.stderr = actual
try:
shell.main(args,**kw)
except SystemExit, e:
except SystemExit as e:
self.assertEqual(code,e.args[0])
else:
self.fail('No exception raised')
......@@ -502,7 +503,7 @@ class TestShellDatabase(Shell, DB):
result = self.env.run('migrate create_model %s %s' % (self.url, repos_path))
temp_dict = dict()
exec result.stdout in temp_dict
six.exec_(result.stdout, temp_dict)
# TODO: breaks on SA06 and SA05 - in need of total refactor - use different approach
......
......@@ -2,7 +2,7 @@
Configuration parser module.
"""
from ConfigParser import ConfigParser
from six.moves.configparser import ConfigParser
from migrate.versioning.config import *
from migrate.versioning import pathed
......
......@@ -9,6 +9,7 @@ http://code.google.com/p/sqlautocode/
import sys
import logging
import six
import sqlalchemy
import migrate
......@@ -68,7 +69,10 @@ class ModelGenerator(object):
# crs: not sure if this is good idea, but it gets rid of extra
# u''
name = col.name.encode('utf8')
if six.PY3:
name = col.name
else:
name = col.name.encode('utf8')
type_ = col.type
for cls in col.type.__class__.__mro__:
......@@ -192,7 +196,7 @@ class ModelGenerator(object):
downgradeCommands.append(
"post_meta.tables[%(table)r].drop()" % {'table': tn})
for (tn, td) in self.diff.tables_different.iteritems():
for (tn, td) in six.iteritems(self.diff.tables_different):
if td.columns_missing_from_A or td.columns_different:
pre_table = self.diff.metadataB.tables[tn]
decls.extend(self._getTableDefn(
......
......@@ -43,7 +43,7 @@ class Changeset(dict):
"""
In a series of upgrades x -> y, keys are version x. Sorted.
"""
ret = super(Changeset, self).keys()
ret = list(super(Changeset, self).keys())
# Reverse order if downgrading
ret.sort(reverse=(self.step < 1))
return ret
......@@ -94,7 +94,7 @@ class Repository(pathed.Pathed):
cls.require_found(path)
cls.require_found(os.path.join(path, cls._config))
cls.require_found(os.path.join(path, cls._versions))
except exceptions.PathNotFoundError, e:
except exceptions.PathNotFoundError:
raise exceptions.InvalidRepositoryError(path)
@classmethod
......@@ -221,7 +221,7 @@ class Repository(pathed.Pathed):
range_mod = 0
op = 'downgrade'
versions = range(start + range_mod, end + range_mod, step)
versions = range(int(start) + range_mod, int(end) + range_mod, step)
changes = [self.version(v).script(database, op) for v in versions]
ret = Changeset(start, step=step, *changes)
return ret
......
......@@ -4,6 +4,7 @@
import sys
import logging
import six
from sqlalchemy import (Table, Column, MetaData, String, Text, Integer,
create_engine)
from sqlalchemy.sql import and_
......@@ -24,7 +25,7 @@ class ControlledSchema(object):
"""A database under version control"""
def __init__(self, engine, repository):
if isinstance(repository, basestring):
if isinstance(repository, six.string_types):
repository = Repository(repository)
self.engine = engine
self.repository = repository
......@@ -49,7 +50,8 @@ class ControlledSchema(object):
data = list(result)[0]
except:
cls, exc, tb = sys.exc_info()
raise exceptions.DatabaseNotControlledError, exc.__str__(), tb
six.reraise(exceptions.DatabaseNotControlledError,
exceptions.DatabaseNotControlledError(str(exc)), tb)
self.version = data['version']
return data
......@@ -133,7 +135,7 @@ class ControlledSchema(object):
"""
# Confirm that the version # is valid: positive, integer,
# exists in repos
if isinstance(repository, basestring):
if isinstance(repository, six.string_types):
repository = Repository(repository)
version = cls._validate_version(repository, version)
table = cls._create_table_version(engine, repository, version)
......@@ -198,7 +200,7 @@ class ControlledSchema(object):
"""
Compare the current model against the current database.
"""
if isinstance(repository, basestring):
if isinstance(repository, six.string_types):
repository = Repository(repository)
model = load_model(model)
......@@ -211,7 +213,7 @@ class ControlledSchema(object):
"""
Dump the current database as a Python model.
"""
if isinstance(repository, basestring):
if isinstance(repository, six.string_types):
repository = Repository(repository)
diff = schemadiff.getDiffOfModelAgainstDatabase(
......
......@@ -99,6 +99,9 @@ class ColDiff(object):
def __nonzero__(self):
return self.diff
__bool__ = __nonzero__
class TableDiff(object):
"""
Container for differences in one :class:`~sqlalchemy.schema.Table`
......@@ -135,6 +138,8 @@ class TableDiff(object):
self.columns_different
)
__bool__ = __nonzero__
class SchemaDiff(object):
"""
Compute the difference between two :class:`~sqlalchemy.schema.MetaData`
......
......@@ -5,7 +5,6 @@ import shutil
import warnings
import logging
import inspect
from StringIO import StringIO
import migrate
from migrate.versioning import genmodel, schemadiff
......@@ -14,6 +13,8 @@ from migrate.versioning.template import Template
from migrate.versioning.script import base
from migrate.versioning.util import import_path, load_model, with_engine
from migrate.exceptions import MigrateDeprecationWarning, InvalidScriptError, ScriptError
import six
from six.moves import StringIO
log = logging.getLogger(__name__)
__all__ = ['PythonScript']
......@@ -51,7 +52,7 @@ class PythonScript(base.BaseScript):
:rtype: string
"""
if isinstance(repository, basestring):
if isinstance(repository, six.string_types):
# oh dear, an import cycle!
from migrate.versioning.repository import Repository
repository = Repository(repository)
......@@ -96,7 +97,7 @@ class PythonScript(base.BaseScript):
module = import_path(path)
try:
assert callable(module.upgrade)
except Exception, e:
except Exception as e:
raise InvalidScriptError(path + ': %s' % str(e))
return module
......@@ -127,7 +128,9 @@ class PythonScript(base.BaseScript):
:type engine: string
:type step: int
"""
if step > 0:
if step in ('downgrade', 'upgrade'):
op = step
elif step > 0:
op = 'upgrade'
elif step < 0:
op = 'downgrade'
......
......@@ -12,6 +12,7 @@ from migrate import exceptions
from migrate.versioning import api
from migrate.versioning.config import *
from migrate.versioning.util import asbool
import six
alias = dict(
......@@ -23,7 +24,7 @@ alias = dict(
def alias_setup():
global alias
for key, val in alias.iteritems():
for key, val in six.iteritems(alias):
setattr(api, key, val)
alias_setup()
......@@ -135,7 +136,7 @@ def main(argv=None, **kwargs):
override_kwargs[opt] = value
# override kwargs with options if user is overwriting
for key, value in options.__dict__.iteritems():
for key, value in six.iteritems(options.__dict__):
if value is not None:
override_kwargs[key] = value
......@@ -143,7 +144,7 @@ def main(argv=None, **kwargs):
f_required = list(f_args)
candidates = dict(kwargs)
candidates.update(override_kwargs)
for key, value in candidates.iteritems():
for key, value in six.iteritems(candidates):
if key in f_args:
f_required.remove(key)
......@@ -160,7 +161,7 @@ def main(argv=None, **kwargs):
kwargs.update(override_kwargs)
# configure options
for key, value in options.__dict__.iteritems():
for key, value in six.iteritems(options.__dict__):
kwargs.setdefault(key, value)