dbmigrate: 2to3 pass with fixes

rhodecode/lib/dbmigrate/migrate/changeset/ansisql.py

@@ -1,314 +1,314 b''
 """
    Extensions to SQLAlchemy for altering existing tables.
 
    At the moment, this isn't so much based off of ANSI as much as
    things that just happen to work with multiple databases.
 """
-from io import StringIO
+import io
 
 import sqlalchemy as sa
 from sqlalchemy.schema import SchemaVisitor
 from sqlalchemy.engine.default import DefaultDialect
 from sqlalchemy.sql import ClauseElement
 from sqlalchemy.schema import (ForeignKeyConstraint,
                                PrimaryKeyConstraint,
                                CheckConstraint,
                                UniqueConstraint,
                                Index)
 
 import sqlalchemy.sql.compiler
 from rhodecode.lib.dbmigrate.migrate import exceptions
 from rhodecode.lib.dbmigrate.migrate.changeset import constraint
 from rhodecode.lib.dbmigrate.migrate.changeset import util
 
 from sqlalchemy.schema import AddConstraint, DropConstraint
 from sqlalchemy.sql.compiler import DDLCompiler
 SchemaGenerator = SchemaDropper = DDLCompiler
 
 
 class AlterTableVisitor(SchemaVisitor):
     """Common operations for ``ALTER TABLE`` statements."""
 
     # engine.Compiler looks for .statement
     # when it spawns off a new compiler
     statement = ClauseElement()
 
     def append(self, s):
         """Append content to the SchemaIterator's query buffer."""
 
         self.buffer.write(s)
 
     def execute(self):
         """Execute the contents of the SchemaIterator's buffer."""
         try:
             return self.connection.execute(self.buffer.getvalue())
         finally:
             self.buffer.seek(0)
             self.buffer.truncate()
 
     def __init__(self, dialect, connection, **kw):
         self.connection = connection
-        self.buffer = StringIO.StringIO()
+        self.buffer = io.StringIO()
         self.preparer = dialect.identifier_preparer
         self.dialect = dialect
 
     def traverse_single(self, elem):
         ret = super(AlterTableVisitor, self).traverse_single(elem)
         if ret:
             # adapt to 0.6 which uses a string-returning
             # object
             self.append(" %s" % ret)
 
     def _to_table(self, param):
         """Returns the table object for the given param object."""
         if isinstance(param, (sa.Column, sa.Index, sa.schema.Constraint)):
             ret = param.table
         else:
             ret = param
         return ret
 
     def start_alter_table(self, param):
         """Returns the start of an ``ALTER TABLE`` SQL-Statement.
 
         Use the param object to determine the table name and use it
         for building the SQL statement.
 
         :param param: object to determine the table from
         :type param: :class:`sqlalchemy.Column`, :class:`sqlalchemy.Index`,
           :class:`sqlalchemy.schema.Constraint`, :class:`sqlalchemy.Table`,
           or string (table name)
         """
         table = self._to_table(param)
         self.append('\nALTER TABLE %s ' % self.preparer.format_table(table))
         return table
 
 
 class ANSIColumnGenerator(AlterTableVisitor, SchemaGenerator):
     """Extends ansisql generator for column creation (alter table add col)"""
 
     def visit_column(self, column):
         """Create a column (table already exists).
 
         :param column: column object
         :type column: :class:`sqlalchemy.Column` instance
         """
         if column.default is not None:
             self.traverse_single(column.default)
 
         table = self.start_alter_table(column)
         self.append("ADD ")
         self.append(self.get_column_specification(column))
 
         for cons in column.constraints:
             self.traverse_single(cons)
         self.execute()
 
         # ALTER TABLE STATEMENTS
 
         # add indexes and unique constraints
         if column.index_name:
             Index(column.index_name,column).create()
         elif column.unique_name:
             constraint.UniqueConstraint(column,
                                         name=column.unique_name).create()
 
         # SA bounds FK constraints to table, add manually
         for fk in column.foreign_keys:
             self.add_foreignkey(fk.constraint)
 
         # add primary key constraint if needed
         if column.primary_key_name:
             cons = constraint.PrimaryKeyConstraint(column,
                                                    name=column.primary_key_name)
             cons.create()
 
     def add_foreignkey(self, fk):
         self.connection.execute(AddConstraint(fk))
 
 class ANSIColumnDropper(AlterTableVisitor, SchemaDropper):
     """Extends ANSI SQL dropper for column dropping (``ALTER TABLE
     DROP COLUMN``).
     """
 
     def visit_column(self, column):
         """Drop a column from its table.
 
         :param column: the column object
         :type column: :class:`sqlalchemy.Column`
         """
         table = self.start_alter_table(column)
         self.append('DROP COLUMN %s' % self.preparer.format_column(column))
         self.execute()
 
 
 class ANSISchemaChanger(AlterTableVisitor, SchemaGenerator):
     """Manages changes to existing schema elements.
 
     Note that columns are schema elements; ``ALTER TABLE ADD COLUMN``
     is in SchemaGenerator.
 
     All items may be renamed. Columns can also have many of their properties -
     type, for example - changed.
 
     Each function is passed a tuple, containing (object, name); where
     object is a type of object you'd expect for that function
     (ie. table for visit_table) and name is the object's new
     name. NONE means the name is unchanged.
     """
 
     def visit_table(self, table):
         """Rename a table. Other ops aren't supported."""
         self.start_alter_table(table)
         q = util.safe_quote(table)
         self.append("RENAME TO %s" % self.preparer.quote(table.new_name, q))
         self.execute()
 
     def visit_index(self, index):
         """Rename an index"""
         if hasattr(self, '_validate_identifier'):
             # SA <= 0.6.3
             self.append("ALTER INDEX %s RENAME TO %s" % (
                     self.preparer.quote(
                         self._validate_identifier(
                             index.name, True), index.quote),
                     self.preparer.quote(
                         self._validate_identifier(
                             index.new_name, True), index.quote)))
         elif hasattr(self, '_index_identifier'):
             # SA >= 0.6.5, < 0.8
             self.append("ALTER INDEX %s RENAME TO %s" % (
                     self.preparer.quote(
                         self._index_identifier(
                             index.name), index.quote),
                     self.preparer.quote(
                         self._index_identifier(
                             index.new_name), index.quote)))
         else:
             # SA >= 0.8
             class NewName(object):
                 """Map obj.name -> obj.new_name"""
                 def __init__(self, index):
                     self.name = index.new_name
                     self._obj = index
 
                 def __getattr__(self, attr):
                     if attr == 'name':
                         return getattr(self, attr)
                     return getattr(self._obj, attr)
 
             self.append("ALTER INDEX %s RENAME TO %s" % (
                     self._prepared_index_name(index),
                     self._prepared_index_name(NewName(index))))
 
         self.execute()
 
     def visit_column(self, delta):
         """Rename/change a column."""
         # ALTER COLUMN is implemented as several ALTER statements
-        keys = delta.keys()
+        keys = list(delta.keys())
         if 'type' in keys:
             self._run_subvisit(delta, self._visit_column_type)
         if 'nullable' in keys:
             self._run_subvisit(delta, self._visit_column_nullable)
         if 'server_default' in keys:
             # Skip 'default': only handle server-side defaults, others
             # are managed by the app, not the db.
             self._run_subvisit(delta, self._visit_column_default)
         if 'name' in keys:
             self._run_subvisit(delta, self._visit_column_name, start_alter=False)
 
     def _run_subvisit(self, delta, func, start_alter=True):
         """Runs visit method based on what needs to be changed on column"""
         table = self._to_table(delta.table)
         col_name = delta.current_name
         if start_alter:
             self.start_alter_column(table, col_name)
         ret = func(table, delta.result_column, delta)
         self.execute()
 
     def start_alter_column(self, table, col_name):
         """Starts ALTER COLUMN"""
         self.start_alter_table(table)
         q = util.safe_quote(table)
         self.append("ALTER COLUMN %s " % self.preparer.quote(col_name, q))
 
     def _visit_column_nullable(self, table, column, delta):
         nullable = delta['nullable']
         if nullable:
             self.append("DROP NOT NULL")
         else:
             self.append("SET NOT NULL")
 
     def _visit_column_default(self, table, column, delta):
         default_text = self.get_column_default_string(column)
         if default_text is not None:
             self.append("SET DEFAULT %s" % default_text)
         else:
             self.append("DROP DEFAULT")
 
     def _visit_column_type(self, table, column, delta):
         type_ = delta['type']
         type_text = str(type_.compile(dialect=self.dialect))
         self.append("TYPE %s" % type_text)
 
     def _visit_column_name(self, table, column, delta):
         self.start_alter_table(table)
         q = util.safe_quote(table)
         col_name = self.preparer.quote(delta.current_name, q)
         new_name = self.preparer.format_column(delta.result_column)
         self.append('RENAME COLUMN %s TO %s' % (col_name, new_name))
 
 
 class ANSIConstraintCommon(AlterTableVisitor):
     """
     Migrate's constraints require a separate creation function from
     SA's: Migrate's constraints are created independently of a table;
     SA's are created at the same time as the table.
     """
 
     def get_constraint_name(self, cons):
         """Gets a name for the given constraint.
 
         If the name is already set it will be used otherwise the
         constraint's :meth:`autoname <migrate.changeset.constraint.ConstraintChangeset.autoname>`
         method is used.
 
         :param cons: constraint object
         """
         if cons.name is not None:
             ret = cons.name
         else:
             ret = cons.name = cons.autoname()
         return self.preparer.quote(ret, cons.quote)
 
     def visit_migrate_primary_key_constraint(self, *p, **k):
         self._visit_constraint(*p, **k)
 
     def visit_migrate_foreign_key_constraint(self, *p, **k):
         self._visit_constraint(*p, **k)
 
     def visit_migrate_check_constraint(self, *p, **k):
         self._visit_constraint(*p, **k)
 
     def visit_migrate_unique_constraint(self, *p, **k):
         self._visit_constraint(*p, **k)
 
 class ANSIConstraintGenerator(ANSIConstraintCommon, SchemaGenerator):
     def _visit_constraint(self, constraint):
         constraint.name = self.get_constraint_name(constraint)
         self.append(self.process(AddConstraint(constraint)))
         self.execute()
 
 class ANSIConstraintDropper(ANSIConstraintCommon, SchemaDropper):
     def _visit_constraint(self, constraint):
         constraint.name = self.get_constraint_name(constraint)
         self.append(self.process(DropConstraint(constraint, cascade=constraint.cascade)))
         self.execute()
 
 
 class ANSIDialect(DefaultDialect):
     columngenerator = ANSIColumnGenerator
     columndropper = ANSIColumnDropper
     schemachanger = ANSISchemaChanger
     constraintgenerator = ANSIConstraintGenerator
     constraintdropper = ANSIConstraintDropper

rhodecode/lib/dbmigrate/migrate/changeset/constraint.py

@@ -1,200 +1,200 b''
 """
    This module defines standalone schema constraint classes.
 """
 from sqlalchemy import schema
 
 from rhodecode.lib.dbmigrate.migrate.exceptions import *
 
 
 class ConstraintChangeset(object):
     """Base class for Constraint classes."""
 
     def _normalize_columns(self, cols, table_name=False):
         """Given: column objects or names; return col names and
         (maybe) a table"""
         colnames = []
         table = None
         for col in cols:
             if isinstance(col, schema.Column):
                 if col.table is not None and table is None:
                     table = col.table
                 if table_name:
                     col = '.'.join((col.table.name, col.name))
                 else:
                     col = col.name
             colnames.append(col)
         return colnames, table
 
     def __do_imports(self, visitor_name, *a, **kw):
         engine = kw.pop('engine', self.table.bind)
         from rhodecode.lib.dbmigrate.migrate.changeset.databases.visitor import (
             get_engine_visitor, run_single_visitor)
         visitorcallable = get_engine_visitor(engine, visitor_name)
         run_single_visitor(engine, visitorcallable, self, *a, **kw)
 
     def create(self, *a, **kw):
         """Create the constraint in the database.
 
         :param engine: the database engine to use. If this is \
         :keyword:`None` the instance's engine will be used
         :type engine: :class:`sqlalchemy.engine.base.Engine`
         :param connection: reuse connection istead of creating new one.
         :type connection: :class:`sqlalchemy.engine.base.Connection` instance
         """
         # TODO: set the parent here instead of in __init__
         self.__do_imports('constraintgenerator', *a, **kw)
 
     def drop(self, *a, **kw):
         """Drop the constraint from the database.
 
         :param engine: the database engine to use. If this is
           :keyword:`None` the instance's engine will be used
         :param cascade: Issue CASCADE drop if database supports it
         :type engine: :class:`sqlalchemy.engine.base.Engine`
         :type cascade: bool
         :param connection: reuse connection istead of creating new one.
         :type connection: :class:`sqlalchemy.engine.base.Connection` instance
         :returns: Instance with cleared columns
         """
         self.cascade = kw.pop('cascade', False)
         self.__do_imports('constraintdropper', *a, **kw)
         # the spirit of Constraint objects is that they
         # are immutable (just like in a DB.  they're only ADDed
         # or DROPped).
         #self.columns.clear()
         return self
 
 
 class PrimaryKeyConstraint(ConstraintChangeset, schema.PrimaryKeyConstraint):
     """Construct PrimaryKeyConstraint
 
     Migrate's additional parameters:
 
     :param cols: Columns in constraint.
     :param table: If columns are passed as strings, this kw is required
     :type table: Table instance
     :type cols: strings or Column instances
     """
 
     __migrate_visit_name__ = 'migrate_primary_key_constraint'
 
     def __init__(self, *cols, **kwargs):
         colnames, table = self._normalize_columns(cols)
         table = kwargs.pop('table', table)
         super(PrimaryKeyConstraint, self).__init__(*colnames, **kwargs)
         if table is not None:
             self._set_parent(table)
 
     def autoname(self):
         """Mimic the database's automatic constraint names"""
         return "%s_pkey" % self.table.name
 
 
 class ForeignKeyConstraint(ConstraintChangeset, schema.ForeignKeyConstraint):
     """Construct ForeignKeyConstraint
 
     Migrate's additional parameters:
 
     :param columns: Columns in constraint
     :param refcolumns: Columns that this FK reffers to in another table.
     :param table: If columns are passed as strings, this kw is required
     :type table: Table instance
     :type columns: list of strings or Column instances
     :type refcolumns: list of strings or Column instances
     """
 
     __migrate_visit_name__ = 'migrate_foreign_key_constraint'
 
     def __init__(self, columns, refcolumns, *args, **kwargs):
         colnames, table = self._normalize_columns(columns)
         table = kwargs.pop('table', table)
         refcolnames, reftable = self._normalize_columns(refcolumns,
                                                         table_name=True)
         super(ForeignKeyConstraint, self).__init__(
             colnames, refcolnames, *args, **kwargs
         )
         if table is not None:
             self._set_parent(table)
 
     @property
     def referenced(self):
         return [e.column for e in self.elements]
 
     @property
     def reftable(self):
         return self.referenced[0].table
 
     def autoname(self):
         """Mimic the database's automatic constraint names"""
         if hasattr(self.columns, 'keys'):
             # SA <= 0.5
-            firstcol = self.columns[self.columns.keys()[0]]
+            firstcol = self.columns[list(self.columns.keys())[0]]
             ret = "%(table)s_%(firstcolumn)s_fkey" % {
                 'table': firstcol.table.name,
                 'firstcolumn': firstcol.name,}
         else:
             # SA >= 0.6
             ret = "%(table)s_%(firstcolumn)s_fkey" % {
                 'table': self.table.name,
                 'firstcolumn': self.columns[0],}
         return ret
 
 
 class CheckConstraint(ConstraintChangeset, schema.CheckConstraint):
     """Construct CheckConstraint
 
     Migrate's additional parameters:
 
     :param sqltext: Plain SQL text to check condition
     :param columns: If not name is applied, you must supply this kw\
     to autoname constraint
     :param table: If columns are passed as strings, this kw is required
     :type table: Table instance
     :type columns: list of Columns instances
     :type sqltext: string
     """
 
     __migrate_visit_name__ = 'migrate_check_constraint'
 
     def __init__(self, sqltext, *args, **kwargs):
         cols = kwargs.pop('columns', [])
         if not cols and not kwargs.get('name', False):
             raise InvalidConstraintError('You must either set "name"'
                 'parameter or "columns" to autogenarate it.')
         colnames, table = self._normalize_columns(cols)
         table = kwargs.pop('table', table)
         schema.CheckConstraint.__init__(self, sqltext, *args, **kwargs)
         if table is not None:
             self._set_parent(table)
         self.colnames = colnames
 
     def autoname(self):
         return "%(table)s_%(cols)s_check" % \
             {'table': self.table.name, 'cols': "_".join(self.colnames)}
 
 
 class UniqueConstraint(ConstraintChangeset, schema.UniqueConstraint):
     """Construct UniqueConstraint
 
     Migrate's additional parameters:
 
     :param cols: Columns in constraint.
     :param table: If columns are passed as strings, this kw is required
     :type table: Table instance
     :type cols: strings or Column instances
 
     .. versionadded:: 0.6.0
     """
 
     __migrate_visit_name__ = 'migrate_unique_constraint'
 
     def __init__(self, *cols, **kwargs):
         self.colnames, table = self._normalize_columns(cols)
         table = kwargs.pop('table', table)
         super(UniqueConstraint, self).__init__(*self.colnames, **kwargs)
         if table is not None:
             self._set_parent(table)
 
     def autoname(self):
         """Mimic the database's automatic constraint names"""
         return "%s_%s_key" % (self.table.name, '_'.join(self.colnames))

rhodecode/lib/dbmigrate/migrate/changeset/databases/oracle.py

@@ -1,108 +1,108 b''
 """
    Oracle database specific implementations of changeset classes.
 """
 import sqlalchemy as sa
 from sqlalchemy.databases import oracle as sa_base
 
 from rhodecode.lib.dbmigrate.migrate import exceptions
 from rhodecode.lib.dbmigrate.migrate.changeset import ansisql
 
 
 OracleSchemaGenerator = sa_base.OracleDDLCompiler
 
 
 class OracleColumnGenerator(OracleSchemaGenerator, ansisql.ANSIColumnGenerator):
     pass
 
 
 class OracleColumnDropper(ansisql.ANSIColumnDropper):
     pass
 
 
 class OracleSchemaChanger(OracleSchemaGenerator, ansisql.ANSISchemaChanger):
 
     def get_column_specification(self, column, **kwargs):
         # Ignore the NOT NULL generated
         override_nullable = kwargs.pop('override_nullable', None)
         if override_nullable:
             orig = column.nullable
             column.nullable = True
         ret = super(OracleSchemaChanger, self).get_column_specification(
             column, **kwargs)
         if override_nullable:
             column.nullable = orig
         return ret
 
     def visit_column(self, delta):
-        keys = delta.keys()
+        keys = list(delta.keys())
 
         if 'name' in keys:
             self._run_subvisit(delta,
                                self._visit_column_name,
                                start_alter=False)
 
         if len(set(('type', 'nullable', 'server_default')).intersection(keys)):
             self._run_subvisit(delta,
                                self._visit_column_change,
                                start_alter=False)
 
     def _visit_column_change(self, table, column, delta):
         # Oracle cannot drop a default once created, but it can set it
         # to null.  We'll do that if default=None
         # http://forums.oracle.com/forums/message.jspa?messageID=1273234#1273234
         dropdefault_hack = (column.server_default is None \
-                                and 'server_default' in delta.keys())
+                                and 'server_default' in list(delta.keys()))
         # Oracle apparently doesn't like it when we say "not null" if
         # the column's already not null. Fudge it, so we don't need a
         # new function
         notnull_hack = ((not column.nullable) \
-                            and ('nullable' not in delta.keys()))
+                            and ('nullable' not in list(delta.keys())))
         # We need to specify NULL if we're removing a NOT NULL
         # constraint
-        null_hack = (column.nullable and ('nullable' in delta.keys()))
+        null_hack = (column.nullable and ('nullable' in list(delta.keys())))
 
         if dropdefault_hack:
             column.server_default = sa.PassiveDefault(sa.sql.null())
         if notnull_hack:
             column.nullable = True
         colspec = self.get_column_specification(column,
             override_nullable=null_hack)
         if null_hack:
             colspec += ' NULL'
         if notnull_hack:
             column.nullable = False
         if dropdefault_hack:
             column.server_default = None
 
         self.start_alter_table(table)
         self.append("MODIFY (")
         self.append(colspec)
         self.append(")")
 
 
 class OracleConstraintCommon(object):
 
     def get_constraint_name(self, cons):
         # Oracle constraints can't guess their name like other DBs
         if not cons.name:
             raise exceptions.NotSupportedError(
                 "Oracle constraint names must be explicitly stated")
         return cons.name
 
 
 class OracleConstraintGenerator(OracleConstraintCommon,
                                 ansisql.ANSIConstraintGenerator):
     pass
 
 
 class OracleConstraintDropper(OracleConstraintCommon,
                               ansisql.ANSIConstraintDropper):
     pass
 
 
 class OracleDialect(ansisql.ANSIDialect):
     columngenerator = OracleColumnGenerator
     columndropper = OracleColumnDropper
     schemachanger = OracleSchemaChanger
     constraintgenerator = OracleConstraintGenerator
     constraintdropper = OracleConstraintDropper

rhodecode/lib/dbmigrate/migrate/changeset/schema.py

@@ -1,668 +1,668 b''
 """
    Schema module providing common schema operations.
 """
 import abc
 try:  # Python 3
     from collections.abc import MutableMapping as DictMixin
 except ImportError:  # Python 2
     from UserDict import DictMixin
 import warnings
 
 import sqlalchemy
 
 from sqlalchemy.schema import ForeignKeyConstraint
 from sqlalchemy.schema import UniqueConstraint
 
 from rhodecode.lib.dbmigrate.migrate.exceptions import *
 from rhodecode.lib.dbmigrate.migrate.changeset import SQLA_07, SQLA_08
 from rhodecode.lib.dbmigrate.migrate.changeset import util
 from rhodecode.lib.dbmigrate.migrate.changeset.databases.visitor import (
     get_engine_visitor, run_single_visitor)
 
 
 __all__ = [
     'create_column',
     'drop_column',
     'alter_column',
     'rename_table',
     'rename_index',
     'ChangesetTable',
     'ChangesetColumn',
     'ChangesetIndex',
     'ChangesetDefaultClause',
     'ColumnDelta',
 ]
 
 def create_column(column, table=None, *p, **kw):
     """Create a column, given the table.
 
     API to :meth:`ChangesetColumn.create`.
     """
     if table is not None:
         return table.create_column(column, *p, **kw)
     return column.create(*p, **kw)
 
 
 def drop_column(column, table=None, *p, **kw):
     """Drop a column, given the table.
 
     API to :meth:`ChangesetColumn.drop`.
     """
     if table is not None:
         return table.drop_column(column, *p, **kw)
     return column.drop(*p, **kw)
 
 
 def rename_table(table, name, engine=None, **kw):
     """Rename a table.
 
     If Table instance is given, engine is not used.
 
     API to :meth:`ChangesetTable.rename`.
 
     :param table: Table to be renamed.
     :param name: New name for Table.
     :param engine: Engine instance.
     :type table: string or Table instance
     :type name: string
     :type engine: obj
     """
     table = _to_table(table, engine)
     table.rename(name, **kw)
 
 
 def rename_index(index, name, table=None, engine=None, **kw):
     """Rename an index.
 
     If Index instance is given,
     table and engine are not used.
 
     API to :meth:`ChangesetIndex.rename`.
 
     :param index: Index to be renamed.
     :param name: New name for index.
     :param table: Table to which Index is reffered.
     :param engine: Engine instance.
     :type index: string or Index instance
     :type name: string
     :type table: string or Table instance
     :type engine: obj
     """
     index = _to_index(index, table, engine)
     index.rename(name, **kw)
 
 
 def alter_column(*p, **k):
     """Alter a column.
 
     This is a helper function that creates a :class:`ColumnDelta` and
     runs it.
 
     :argument column:
       The name of the column to be altered or a
       :class:`ChangesetColumn` column representing it.
 
     :param table:
       A :class:`~sqlalchemy.schema.Table` or table name to
       for the table where the column will be changed.
 
     :param engine:
       The :class:`~sqlalchemy.engine.base.Engine` to use for table
       reflection and schema alterations.
 
     :returns: A :class:`ColumnDelta` instance representing the change.
 
 
     """
 
     if 'table' not in k and isinstance(p[0], sqlalchemy.Column):
         k['table'] = p[0].table
     if 'engine' not in k:
         k['engine'] = k['table'].bind
 
     # deprecation
     if len(p) >= 2 and isinstance(p[1], sqlalchemy.Column):
         warnings.warn(
             "Passing a Column object to alter_column is deprecated."
             " Just pass in keyword parameters instead.",
             MigrateDeprecationWarning
             )
     engine = k['engine']
 
     # enough tests seem to break when metadata is always altered
     # that this crutch has to be left in until they can be sorted
     # out
     k['alter_metadata']=True
 
     delta = ColumnDelta(*p, **k)
 
     visitorcallable = get_engine_visitor(engine, 'schemachanger')
     engine._run_visitor(visitorcallable, delta)
 
     return delta
 
 
 def _to_table(table, engine=None):
     """Return if instance of Table, else construct new with metadata"""
     if isinstance(table, sqlalchemy.Table):
         return table
 
     # Given: table name, maybe an engine
     meta = sqlalchemy.MetaData()
     if engine is not None:
         meta.bind = engine
     return sqlalchemy.Table(table, meta)
 
 
 def _to_index(index, table=None, engine=None):
     """Return if instance of Index, else construct new with metadata"""
     if isinstance(index, sqlalchemy.Index):
         return index
 
     # Given: index name; table name required
     table = _to_table(table, engine)
     ret = sqlalchemy.Index(index)
     ret.table = table
     return ret
 
 
 class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem):
     """Extracts the differences between two columns/column-parameters
 
         May receive parameters arranged in several different ways:
 
         * **current_column, new_column, \*p, \*\*kw**
             Additional parameters can be specified to override column
             differences.
 
         * **current_column, \*p, \*\*kw**
             Additional parameters alter current_column. Table name is extracted
             from current_column object.
             Name is changed to current_column.name from current_name,
             if current_name is specified.
 
         * **current_col_name, \*p, \*\*kw**
             Table kw must specified.
 
         :param table: Table at which current Column should be bound to.\
         If table name is given, reflection will be used.
         :type table: string or Table instance
 
         :param metadata: A :class:`MetaData` instance to store
                          reflected table names
 
         :param engine: When reflecting tables, either engine or metadata must \
         be specified to acquire engine object.
         :type engine: :class:`Engine` instance
         :returns: :class:`ColumnDelta` instance provides interface for altered attributes to \
         `result_column` through :func:`dict` alike object.
 
         * :class:`ColumnDelta`.result_column is altered column with new attributes
 
         * :class:`ColumnDelta`.current_name is current name of column in db
 
 
     """
 
     # Column attributes that can be altered
     diff_keys = ('name', 'type', 'primary_key', 'nullable',
         'server_onupdate', 'server_default', 'autoincrement')
     diffs = dict()
     __visit_name__ = 'column'
 
     def __init__(self, *p, **kw):
         # 'alter_metadata' is not a public api. It exists purely
         # as a crutch until the tests that fail when 'alter_metadata'
         # behaviour always happens can be sorted out
         self.alter_metadata = kw.pop("alter_metadata", False)
 
         self.meta = kw.pop("metadata", None)
         self.engine = kw.pop("engine", None)
 
         # Things are initialized differently depending on how many column
         # parameters are given. Figure out how many and call the appropriate
         # method.
         if len(p) >= 1 and isinstance(p[0], sqlalchemy.Column):
             # At least one column specified
             if len(p) >= 2 and isinstance(p[1], sqlalchemy.Column):
                 # Two columns specified
                 diffs = self.compare_2_columns(*p, **kw)
             else:
                 # Exactly one column specified
                 diffs = self.compare_1_column(*p, **kw)
         else:
             # Zero columns specified
             if not len(p) or not isinstance(p[0], str):
                 raise ValueError("First argument must be column name")
             diffs = self.compare_parameters(*p, **kw)
 
         self.apply_diffs(diffs)
 
     def __repr__(self):
         return '<ColumnDelta altermetadata=%r, %s>' % (
             self.alter_metadata,
             super(ColumnDelta, self).__repr__()
             )
 
     def __getitem__(self, key):
-        if key not in self.keys():
+        if key not in list(self.keys()):
             raise KeyError("No such diff key, available: %s" % self.diffs )
         return getattr(self.result_column, key)
 
     def __setitem__(self, key, value):
-        if key not in self.keys():
+        if key not in list(self.keys()):
             raise KeyError("No such diff key, available: %s" % self.diffs )
         setattr(self.result_column, key, value)
 
     def __delitem__(self, key):
         raise NotImplementedError
 
     def __len__(self):
         raise NotImplementedError
 
     def __iter__(self):
         raise NotImplementedError
 
     def keys(self):
-        return self.diffs.keys()
+        return list(self.diffs.keys())
 
     def compare_parameters(self, current_name, *p, **k):
         """Compares Column objects with reflection"""
         self.table = k.pop('table')
         self.result_column = self._table.c.get(current_name)
         if len(p):
             k = self._extract_parameters(p, k, self.result_column)
         return k
 
     def compare_1_column(self, col, *p, **k):
         """Compares one Column object"""
         self.table = k.pop('table', None)
         if self.table is None:
             self.table = col.table
         self.result_column = col
         if len(p):
             k = self._extract_parameters(p, k, self.result_column)
         return k
 
     def compare_2_columns(self, old_col, new_col, *p, **k):
         """Compares two Column objects"""
         self.process_column(new_col)
         self.table = k.pop('table', None)
         # we cannot use bool() on table in SA06
         if self.table is None:
             self.table = old_col.table
         if self.table is None:
             new_col.table
         self.result_column = old_col
 
         # set differences
         # leave out some stuff for later comp
         for key in (set(self.diff_keys) - set(('type',))):
             val = getattr(new_col, key, None)
             if getattr(self.result_column, key, None) != val:
                 k.setdefault(key, val)
 
         # inspect types
         if not self.are_column_types_eq(self.result_column.type, new_col.type):
             k.setdefault('type', new_col.type)
 
         if len(p):
             k = self._extract_parameters(p, k, self.result_column)
         return k
 
     def apply_diffs(self, diffs):
         """Populate dict and column object with new values"""
         self.diffs = diffs
         for key in self.diff_keys:
             if key in diffs:
                 setattr(self.result_column, key, diffs[key])
 
         self.process_column(self.result_column)
 
         # create an instance of class type if not yet
         if 'type' in diffs and callable(self.result_column.type):
             self.result_column.type = self.result_column.type()
 
         # add column to the table
         if self.table is not None and self.alter_metadata:
             self.result_column.add_to_table(self.table)
 
     def are_column_types_eq(self, old_type, new_type):
         """Compares two types to be equal"""
         ret = old_type.__class__ == new_type.__class__
 
         # String length is a special case
         if ret and isinstance(new_type, sqlalchemy.types.String):
             ret = (getattr(old_type, 'length', None) == \
                        getattr(new_type, 'length', None))
         return ret
 
     def _extract_parameters(self, p, k, column):
         """Extracts data from p and modifies diffs"""
         p = list(p)
         while len(p):
             if isinstance(p[0], str):
                 k.setdefault('name', p.pop(0))
             elif isinstance(p[0], sqlalchemy.types.TypeEngine):
                 k.setdefault('type', p.pop(0))
             elif callable(p[0]):
                 p[0] = p[0]()
             else:
                 break
 
         if len(p):
             new_col = column.copy_fixed()
             new_col._init_items(*p)
             k = self.compare_2_columns(column, new_col, **k)
         return k
 
     def process_column(self, column):
         """Processes default values for column"""
         # XXX: this is a snippet from SA processing of positional parameters
         toinit = list()
 
         if column.server_default is not None:
             if isinstance(column.server_default, sqlalchemy.FetchedValue):
                 toinit.append(column.server_default)
             else:
                 toinit.append(sqlalchemy.DefaultClause(column.server_default))
         if column.server_onupdate is not None:
             if isinstance(column.server_onupdate, FetchedValue):
                 toinit.append(column.server_default)
             else:
                 toinit.append(sqlalchemy.DefaultClause(column.server_onupdate,
                                             for_update=True))
         if toinit:
             column._init_items(*toinit)
 
     def _get_table(self):
         return getattr(self, '_table', None)
 
     def _set_table(self, table):
         if isinstance(table, str):
             if self.alter_metadata:
                 if not self.meta:
                     raise ValueError("metadata must be specified for table"
                         " reflection when using alter_metadata")
                 meta = self.meta
                 if self.engine:
                     meta.bind = self.engine
             else:
                 if not self.engine and not self.meta:
                     raise ValueError("engine or metadata must be specified"
                         " to reflect tables")
                 if not self.engine:
                     self.engine = self.meta.bind
                 meta = sqlalchemy.MetaData(bind=self.engine)
             self._table = sqlalchemy.Table(table, meta, autoload=True)
         elif isinstance(table, sqlalchemy.Table):
             self._table = table
             if not self.alter_metadata:
                 self._table.meta = sqlalchemy.MetaData(bind=self._table.bind)
     def _get_result_column(self):
         return getattr(self, '_result_column', None)
 
     def _set_result_column(self, column):
         """Set Column to Table based on alter_metadata evaluation."""
         self.process_column(column)
         if not hasattr(self, 'current_name'):
             self.current_name = column.name
         if self.alter_metadata:
             self._result_column = column
         else:
             self._result_column = column.copy_fixed()
 
     table = property(_get_table, _set_table)
     result_column = property(_get_result_column, _set_result_column)
 
 
 class ChangesetTable(object):
     """Changeset extensions to SQLAlchemy tables."""
 
     def create_column(self, column, *p, **kw):
         """Creates a column.
 
         The column parameter may be a column definition or the name of
         a column in this table.
 
         API to :meth:`ChangesetColumn.create`
 
         :param column: Column to be created
         :type column: Column instance or string
         """
         if not isinstance(column, sqlalchemy.Column):
             # It's a column name
             column = getattr(self.c, str(column))
         column.create(table=self, *p, **kw)
 
     def drop_column(self, column, *p, **kw):
         """Drop a column, given its name or definition.
 
         API to :meth:`ChangesetColumn.drop`
 
         :param column: Column to be droped
         :type column: Column instance or string
         """
         if not isinstance(column, sqlalchemy.Column):
             # It's a column name
             try:
                 column = getattr(self.c, str(column))
             except AttributeError:
                 # That column isn't part of the table. We don't need
                 # its entire definition to drop the column, just its
                 # name, so create a dummy column with the same name.
                 column = sqlalchemy.Column(str(column), sqlalchemy.Integer())
         column.drop(table=self, *p, **kw)
 
     def rename(self, name, connection=None, **kwargs):
         """Rename this table.
 
         :param name: New name of the table.
         :type name: string
         :param connection: reuse connection istead of creating new one.
         :type connection: :class:`sqlalchemy.engine.base.Connection` instance
         """
         engine = self.bind
         self.new_name = name
         visitorcallable = get_engine_visitor(engine, 'schemachanger')
         run_single_visitor(engine, visitorcallable, self, connection, **kwargs)
 
         # Fix metadata registration
         self.name = name
         self.deregister()
         self._set_parent(self.metadata)
 
     def _meta_key(self):
         """Get the meta key for this table."""
         return sqlalchemy.schema._get_table_key(self.name, self.schema)
 
     def deregister(self):
         """Remove this table from its metadata"""
         if SQLA_07:
             self.metadata._remove_table(self.name, self.schema)
         else:
             key = self._meta_key()
             meta = self.metadata
             if key in meta.tables:
                 del meta.tables[key]
 
 
 class ChangesetColumn(object):
     """Changeset extensions to SQLAlchemy columns."""
 
     def alter(self, *p, **k):
         """Makes a call to :func:`alter_column` for the column this
         method is called on.
         """
         if 'table' not in k:
             k['table'] = self.table
         if 'engine' not in k:
             k['engine'] = k['table'].bind
         return alter_column(self, *p, **k)
 
     def create(self, table=None, index_name=None, unique_name=None,
                primary_key_name=None, populate_default=True, connection=None, **kwargs):
         """Create this column in the database.
 
         Assumes the given table exists. ``ALTER TABLE ADD COLUMN``,
         for most databases.
 
         :param table: Table instance to create on.
         :param index_name: Creates :class:`ChangesetIndex` on this column.
         :param unique_name: Creates :class:\
 `~migrate.changeset.constraint.UniqueConstraint` on this column.
         :param primary_key_name: Creates :class:\
 `~migrate.changeset.constraint.PrimaryKeyConstraint` on this column.
         :param populate_default: If True, created column will be \
 populated with defaults
         :param connection: reuse connection istead of creating new one.
         :type table: Table instance
         :type index_name: string
         :type unique_name: string
         :type primary_key_name: string
         :type populate_default: bool
         :type connection: :class:`sqlalchemy.engine.base.Connection` instance
 
         :returns: self
         """
         self.populate_default = populate_default
         self.index_name = index_name
         self.unique_name = unique_name
         self.primary_key_name = primary_key_name
         for cons in ('index_name', 'unique_name', 'primary_key_name'):
             self._check_sanity_constraints(cons)
 
         self.add_to_table(table)
         engine = self.table.bind
         visitorcallable = get_engine_visitor(engine, 'columngenerator')
         engine._run_visitor(visitorcallable, self, connection, **kwargs)
 
         # TODO: reuse existing connection
         if self.populate_default and self.default is not None:
             stmt = table.update().values({self: engine._execute_default(self.default)})
             engine.execute(stmt)
 
         return self
 
     def drop(self, table=None, connection=None, **kwargs):
         """Drop this column from the database, leaving its table intact.
 
         ``ALTER TABLE DROP COLUMN``, for most databases.
 
         :param connection: reuse connection istead of creating new one.
         :type connection: :class:`sqlalchemy.engine.base.Connection` instance
         """
         if table is not None:
             self.table = table
         engine = self.table.bind
         visitorcallable = get_engine_visitor(engine, 'columndropper')
         engine._run_visitor(visitorcallable, self, connection, **kwargs)
         self.remove_from_table(self.table, unset_table=False)
         self.table = None
         return self
 
     def add_to_table(self, table):
         if table is not None  and self.table is None:
             if SQLA_07:
                 table.append_column(self)
             else:
                 self._set_parent(table)
 
     def _col_name_in_constraint(self,cons,name):
         return False
 
     def remove_from_table(self, table, unset_table=True):
         # TODO: remove primary keys, constraints, etc
         if unset_table:
             self.table = None
 
         to_drop = set()
         for index in table.indexes:
             columns = []
             for col in index.columns:
                 if col.name!=self.name:
                     columns.append(col)
             if columns:
                 index.columns = columns
                 if SQLA_08:
                     index.expressions = columns
             else:
                 to_drop.add(index)
         table.indexes = table.indexes - to_drop
 
         to_drop = set()
         for cons in table.constraints:
             # TODO: deal with other types of constraint
             if isinstance(cons,(ForeignKeyConstraint,
                                 UniqueConstraint)):
                 for col_name in cons.columns:
                     if not isinstance(col_name, str):
                         col_name = col_name.name
                     if self.name==col_name:
                         to_drop.add(cons)
         table.constraints = table.constraints - to_drop
 
         if table.c.contains_column(self):
             if SQLA_07:
                 table._columns.remove(self)
             else:
                 table.c.remove(self)
 
     # TODO: this is fixed in 0.6
     def copy_fixed(self, **kw):
         """Create a copy of this ``Column``, with all attributes."""
         q = util.safe_quote(self)
         return sqlalchemy.Column(self.name, self.type, self.default,
             key=self.key,
             primary_key=self.primary_key,
             nullable=self.nullable,
             quote=q,
             index=self.index,
             unique=self.unique,
             onupdate=self.onupdate,
             autoincrement=self.autoincrement,
             server_default=self.server_default,
             server_onupdate=self.server_onupdate,
             *[c.copy(**kw) for c in self.constraints])
 
     def _check_sanity_constraints(self, name):
         """Check if constraints names are correct"""
         obj = getattr(self, name)
         if (getattr(self, name[:-5]) and not obj):
             raise InvalidConstraintError("Column.create() accepts index_name,"
             " primary_key_name and unique_name to generate constraints")
         if not isinstance(obj, str) and obj is not None:
             raise InvalidConstraintError(
             "%s argument for column must be constraint name" % name)
 
 
 class ChangesetIndex(object):
     """Changeset extensions to SQLAlchemy Indexes."""
 
     __visit_name__ = 'index'
 
     def rename(self, name, connection=None, **kwargs):
         """Change the name of an index.
 
         :param name: New name of the Index.
         :type name: string
         :param connection: reuse connection istead of creating new one.
         :type connection: :class:`sqlalchemy.engine.base.Connection` instance
         """
         engine = self.table.bind
         self.new_name = name
         visitorcallable = get_engine_visitor(engine, 'schemachanger')
         engine._run_visitor(visitorcallable, self, connection, **kwargs)
         self.name = name
 
 
 class ChangesetDefaultClause(object):
     """Implements comparison between :class:`DefaultClause` instances"""
 
     def __eq__(self, other):
         if isinstance(other, self.__class__):
             if self.arg == other.arg:
                 return True
 
     def __ne__(self, other):
         return not self.__eq__(other)

rhodecode/lib/dbmigrate/migrate/versioning/api.py

@@ -1,386 +1,386 b''
 """
    This module provides an external API to the versioning system.
 
    .. versionchanged:: 0.6.0
     :func:`migrate.versioning.api.test` and schema diff functions
     changed order of positional arguments so all accept `url` and `repository`
     as first arguments.
 
    .. versionchanged:: 0.5.4
     ``--preview_sql`` displays source file when using SQL scripts.
     If Python script is used, it runs the action with mocked engine and
     returns captured SQL statements.
 
    .. versionchanged:: 0.5.4
     Deprecated ``--echo`` parameter in favour of new
     :func:`migrate.versioning.util.construct_engine` behavior.
 """
 
 # Dear migrate developers,
 #
 # please do not comment this module using sphinx syntax because its
 # docstrings are presented as user help and most users cannot
 # interpret sphinx annotated ReStructuredText.
 #
 # Thanks,
 # Jan Dittberner
 
 import sys
 import inspect
 import logging
 
 from rhodecode.lib.dbmigrate.migrate import exceptions
 from rhodecode.lib.dbmigrate.migrate.versioning import (
     repository, schema, version,
     script as script_  # command name conflict
 )
 from rhodecode.lib.dbmigrate.migrate.versioning.util import (
     catch_known_errors, with_engine)
 
 
 log = logging.getLogger(__name__)
 command_desc = {
     'help': 'displays help on a given command',
     'create': 'create an empty repository at the specified path',
     'script': 'create an empty change Python script',
     'script_sql': 'create empty change SQL scripts for given database',
     'version': 'display the latest version available in a repository',
     'db_version': 'show the current version of the repository under version control',
     'source': 'display the Python code for a particular version in this repository',
     'version_control': 'mark a database as under this repository\'s version control',
     'upgrade': 'upgrade a database to a later version',
     'downgrade': 'downgrade a database to an earlier version',
     'drop_version_control': 'removes version control from a database',
     'manage': 'creates a Python script that runs Migrate with a set of default values',
     'test': 'performs the upgrade and downgrade command on the given database',
     'compare_model_to_db': 'compare MetaData against the current database state',
     'create_model': 'dump the current database as a Python model to stdout',
     'make_update_script_for_model': 'create a script changing the old MetaData to the new (current) MetaData',
     'update_db_from_model': 'modify the database to match the structure of the current MetaData',
 }
-__all__ = command_desc.keys()
+__all__ = list(command_desc.keys())
 
 Repository = repository.Repository
 ControlledSchema = schema.ControlledSchema
 VerNum = version.VerNum
 PythonScript = script_.PythonScript
 SqlScript = script_.SqlScript
 
 
 # deprecated
 def help(cmd=None, **opts):
     """%prog help COMMAND
 
     Displays help on a given command.
     """
     if cmd is None:
         raise exceptions.UsageError(None)
     try:
         func = globals()[cmd]
     except:
         raise exceptions.UsageError(
             "'%s' isn't a valid command. Try 'help COMMAND'" % cmd)
     ret = func.__doc__
     if sys.argv[0]:
         ret = ret.replace('%prog', sys.argv[0])
     return ret
 
 @catch_known_errors
 def create(repository, name, **opts):
     """%prog create REPOSITORY_PATH NAME [--table=TABLE]
 
     Create an empty repository at the specified path.
 
     You can specify the version_table to be used; by default, it is
     'migrate_version'.  This table is created in all version-controlled
     databases.
     """
     repo_path = Repository.create(repository, name, **opts)
 
 
 @catch_known_errors
 def script(description, repository, **opts):
     """%prog script DESCRIPTION REPOSITORY_PATH
 
     Create an empty change script using the next unused version number
     appended with the given description.
 
     For instance, manage.py script "Add initial tables" creates:
     repository/versions/001_Add_initial_tables.py
     """
     repo = Repository(repository)
     repo.create_script(description, **opts)
 
 
 @catch_known_errors
 def script_sql(database, description, repository, **opts):
     """%prog script_sql DATABASE DESCRIPTION REPOSITORY_PATH
 
     Create empty change SQL scripts for given DATABASE, where DATABASE
     is either specific ('postgresql', 'mysql', 'oracle', 'sqlite', etc.)
     or generic ('default').
 
     For instance, manage.py script_sql postgresql description creates:
     repository/versions/001_description_postgresql_upgrade.sql and
     repository/versions/001_description_postgresql_downgrade.sql
     """
     repo = Repository(repository)
     repo.create_script_sql(database, description, **opts)
 
 
 def version(repository, **opts):
     """%prog version REPOSITORY_PATH
 
     Display the latest version available in a repository.
     """
     repo = Repository(repository)
     return repo.latest
 
 
 @with_engine
 def db_version(url, repository, **opts):
     """%prog db_version URL REPOSITORY_PATH
 
     Show the current version of the repository with the given
     connection string, under version control of the specified
     repository.
 
     The url should be any valid SQLAlchemy connection string.
     """
     engine = opts.pop('engine')
     schema = ControlledSchema(engine, repository)
     return schema.version
 
 
 def source(version, dest=None, repository=None, **opts):
     """%prog source VERSION [DESTINATION] --repository=REPOSITORY_PATH
 
     Display the Python code for a particular version in this
     repository.  Save it to the file at DESTINATION or, if omitted,
     send to stdout.
     """
     if repository is None:
         raise exceptions.UsageError("A repository must be specified")
     repo = Repository(repository)
     ret = repo.version(version).script().source()
     if dest is not None:
         with open(dest, 'w') as f:
             f.write(ret)
         ret = None
     return ret
 
 
 def upgrade(url, repository, version=None, **opts):
     """%prog upgrade URL REPOSITORY_PATH [VERSION] [--preview_py|--preview_sql]
 
     Upgrade a database to a later version.
 
     This runs the upgrade() function defined in your change scripts.
 
     By default, the database is updated to the latest available
     version. You may specify a version instead, if you wish.
 
     You may preview the Python or SQL code to be executed, rather than
     actually executing it, using the appropriate 'preview' option.
     """
     err = "Cannot upgrade a database of version %s to version %s. "\
         "Try 'downgrade' instead."
     return _migrate(url, repository, version, upgrade=True, err=err, **opts)
 
 
 def downgrade(url, repository, version, **opts):
     """%prog downgrade URL REPOSITORY_PATH VERSION [--preview_py|--preview_sql]
 
     Downgrade a database to an earlier version.
 
     This is the reverse of upgrade; this runs the downgrade() function
     defined in your change scripts.
 
     You may preview the Python or SQL code to be executed, rather than
     actually executing it, using the appropriate 'preview' option.
     """
     err = "Cannot downgrade a database of version %s to version %s. "\
         "Try 'upgrade' instead."
     return _migrate(url, repository, version, upgrade=False, err=err, **opts)
 
 @with_engine
 def test(url, repository, **opts):
     """%prog test URL REPOSITORY_PATH [VERSION]
 
     Performs the upgrade and downgrade option on the given
     database. This is not a real test and may leave the database in a
     bad state. You should therefore better run the test on a copy of
     your database.
     """
     engine = opts.pop('engine')
     repos = Repository(repository)
 
     # Upgrade
     log.info("Upgrading...")
     script = repos.version(None).script(engine.name, 'upgrade')
     script.run(engine, 1)
     log.info("done")
 
     log.info("Downgrading...")
     script = repos.version(None).script(engine.name, 'downgrade')
     script.run(engine, -1)
     log.info("done")
     log.info("Success")
 
 
 @with_engine
 def version_control(url, repository, version=None, **opts):
     """%prog version_control URL REPOSITORY_PATH [VERSION]
 
     Mark a database as under this repository's version control.
 
     Once a database is under version control, schema changes should
     only be done via change scripts in this repository.
 
     This creates the table version_table in the database.
 
     The url should be any valid SQLAlchemy connection string.
 
     By default, the database begins at version 0 and is assumed to be
     empty.  If the database is not empty, you may specify a version at
     which to begin instead. No attempt is made to verify this
     version's correctness - the database schema is expected to be
     identical to what it would be if the database were created from
     scratch.
     """
     engine = opts.pop('engine')
     ControlledSchema.create(engine, repository, version)
 
 
 @with_engine
 def drop_version_control(url, repository, **opts):
     """%prog drop_version_control URL REPOSITORY_PATH
 
     Removes version control from a database.
     """
     engine = opts.pop('engine')
     schema = ControlledSchema(engine, repository)
     schema.drop()
 
 
 def manage(file, **opts):
     """%prog manage FILENAME [VARIABLES...]
 
     Creates a script that runs Migrate with a set of default values.
 
     For example::
 
         %prog manage manage.py --repository=/path/to/repository \
 --url=sqlite:///project.db
 
     would create the script manage.py. The following two commands
     would then have exactly the same results::
 
         python manage.py version
         %prog version --repository=/path/to/repository
     """
     Repository.create_manage_file(file, **opts)
 
 
 @with_engine
 def compare_model_to_db(url, repository, model, **opts):
     """%prog compare_model_to_db URL REPOSITORY_PATH MODEL
 
     Compare the current model (assumed to be a module level variable
     of type sqlalchemy.MetaData) against the current database.
 
     NOTE: This is EXPERIMENTAL.
     """  # TODO: get rid of EXPERIMENTAL label
     engine = opts.pop('engine')
     return ControlledSchema.compare_model_to_db(engine, model, repository)
 
 
 @with_engine
 def create_model(url, repository, **opts):
     """%prog create_model URL REPOSITORY_PATH [DECLERATIVE=True]
 
     Dump the current database as a Python model to stdout.
 
     NOTE: This is EXPERIMENTAL.
     """  # TODO: get rid of EXPERIMENTAL label
     engine = opts.pop('engine')
     declarative = opts.get('declarative', False)
     return ControlledSchema.create_model(engine, repository, declarative)
 
 
 @catch_known_errors
 @with_engine
 def make_update_script_for_model(url, repository, oldmodel, model, **opts):
     """%prog make_update_script_for_model URL OLDMODEL MODEL REPOSITORY_PATH
 
     Create a script changing the old Python model to the new (current)
     Python model, sending to stdout.
 
     NOTE: This is EXPERIMENTAL.
     """  # TODO: get rid of EXPERIMENTAL label
     engine = opts.pop('engine')
     return PythonScript.make_update_script_for_model(
         engine, oldmodel, model, repository, **opts)
 
 
 @with_engine
 def update_db_from_model(url, repository, model, **opts):
     """%prog update_db_from_model URL REPOSITORY_PATH MODEL
 
     Modify the database to match the structure of the current Python
     model. This also sets the db_version number to the latest in the
     repository.
 
     NOTE: This is EXPERIMENTAL.
     """  # TODO: get rid of EXPERIMENTAL label
     engine = opts.pop('engine')
     schema = ControlledSchema(engine, repository)
     schema.update_db_from_model(model)
 
 @with_engine
 def _migrate(url, repository, version, upgrade, err, **opts):
     engine = opts.pop('engine')
     url = str(engine.url)
     schema = ControlledSchema(engine, repository)
     version = _migrate_version(schema, version, upgrade, err)
 
     changeset = schema.changeset(version)
     for ver, change in changeset:
         nextver = ver + changeset.step
         log.info('%s -> %s... ', ver, nextver)
 
         if opts.get('preview_sql'):
             if isinstance(change, PythonScript):
                 log.info(change.preview_sql(url, changeset.step, **opts))
             elif isinstance(change, SqlScript):
                 log.info(change.source())
 
         elif opts.get('preview_py'):
             if not isinstance(change, PythonScript):
                 raise exceptions.UsageError("Python source can be only displayed"
                     " for python migration files")
             source_ver = max(ver, nextver)
             module = schema.repository.version(source_ver).script().module
             funcname = upgrade and "upgrade" or "downgrade"
             func = getattr(module, funcname)
             log.info(inspect.getsource(func))
         else:
             schema.runchange(ver, change, changeset.step)
             log.info('done')
 
 
 def _migrate_version(schema, version, upgrade, err):
     if version is None:
         return version
     # Version is specified: ensure we're upgrading in the right direction
     # (current version < target version for upgrading; reverse for down)
     version = VerNum(version)
     cur = schema.version
     if upgrade is not None:
         if upgrade:
             direction = cur <= version
         else:
             direction = cur >= version
         if not direction:
             raise exceptions.KnownError(err % (cur, version))
     return version

rhodecode/lib/dbmigrate/migrate/versioning/genmodel.py

@@ -1,302 +1,302 b''
 """
 Code to generate a Python model from a database or differences
 between a model and database.
 
 Some of this is borrowed heavily from the AutoCode project at:
 http://code.google.com/p/sqlautocode/
 """
 
 import sys
 import logging
 
 import sqlalchemy
 
 import rhodecode.lib.dbmigrate.migrate
 import rhodecode.lib.dbmigrate.migrate.changeset
 
 
 log = logging.getLogger(__name__)
 HEADER = """
 ## File autogenerated by genmodel.py
 
 from sqlalchemy import *
 """
 
 META_DEFINITION = "meta = MetaData()"
 
 DECLARATIVE_DEFINITION = """
 from sqlalchemy.ext import declarative
 
 Base = declarative.declarative_base()
 """
 
 
 class ModelGenerator(object):
     """Various transformations from an A, B diff.
 
     In the implementation, A tends to be called the model and B
     the database (although this is not true of all diffs).
     The diff is directionless, but transformations apply the diff
     in a particular direction, described in the method name.
     """
 
     def __init__(self, diff, engine, declarative=False):
         self.diff = diff
         self.engine = engine
         self.declarative = declarative
 
     def column_repr(self, col):
         kwarg = []
         if col.key != col.name:
             kwarg.append('key')
         if col.primary_key:
             col.primary_key = True  # otherwise it dumps it as 1
             kwarg.append('primary_key')
         if not col.nullable:
             kwarg.append('nullable')
         if col.onupdate:
             kwarg.append('onupdate')
         if col.default:
             if col.primary_key:
                 # I found that PostgreSQL automatically creates a
                 # default value for the sequence, but let's not show
                 # that.
                 pass
             else:
                 kwarg.append('default')
         args = ['%s=%r' % (k, getattr(col, k)) for k in kwarg]
 
         # crs: not sure if this is good idea, but it gets rid of extra
         # u''
         name = col.name.encode('utf8')
 
         type_ = col.type
         for cls in col.type.__class__.__mro__:
             if cls.__module__ == 'sqlalchemy.types' and \
                 not cls.__name__.isupper():
                 if cls is not type_.__class__:
                     type_ = cls()
                 break
 
         type_repr = repr(type_)
         if type_repr.endswith('()'):
             type_repr = type_repr[:-2]
 
         constraints = [repr(cn) for cn in col.constraints]
 
         data = {
             'name': name,
             'commonStuff': ', '.join([type_repr] + constraints + args),
         }
 
         if self.declarative:
             return """%(name)s = Column(%(commonStuff)s)""" % data
         else:
             return """Column(%(name)r, %(commonStuff)s)""" % data
 
     def _getTableDefn(self, table, metaName='meta'):
         out = []
         tableName = table.name
         if self.declarative:
             out.append("class %(table)s(Base):" % {'table': tableName})
             out.append("    __tablename__ = '%(table)s'\n" %
                             {'table': tableName})
             for col in table.columns:
                 out.append("    %s" % self.column_repr(col))
             out.append('\n')
         else:
             out.append("%(table)s = Table('%(table)s', %(meta)s," %
                        {'table': tableName, 'meta': metaName})
             for col in table.columns:
                 out.append("    %s," % self.column_repr(col))
             out.append(")\n")
         return out
 
     def _get_tables(self,missingA=False,missingB=False,modified=False):
         to_process = []
         for bool_,names,metadata in (
             (missingA,self.diff.tables_missing_from_A,self.diff.metadataB),
             (missingB,self.diff.tables_missing_from_B,self.diff.metadataA),
             (modified,self.diff.tables_different,self.diff.metadataA),
                 ):
             if bool_:
                 for name in names:
                     yield metadata.tables.get(name)
 
     def _genModelHeader(self, tables):
         out = []
         import_index = []
 
         out.append(HEADER)
 
         for table in tables:
             for col in table.columns:
                 if "dialects" in col.type.__module__ and \
                    col.type.__class__ not in import_index:
                     out.append("from " + col.type.__module__ +
                                " import " + col.type.__class__.__name__)
                     import_index.append(col.type.__class__)
 
         out.append("")
 
         if self.declarative:
             out.append(DECLARATIVE_DEFINITION)
         else:
             out.append(META_DEFINITION)
         out.append("")
 
         return out
 
     def genBDefinition(self):
         """Generates the source code for a definition of B.
 
         Assumes a diff where A is empty.
 
         Was: toPython. Assume database (B) is current and model (A) is empty.
         """
 
         out = []
         out.extend(self._genModelHeader(self._get_tables(missingA=True)))
         for table in self._get_tables(missingA=True):
             out.extend(self._getTableDefn(table))
         return '\n'.join(out)
 
     def genB2AMigration(self, indent='    '):
         """Generate a migration from B to A.
 
         Was: toUpgradeDowngradePython
         Assume model (A) is most current and database (B) is out-of-date.
         """
 
         decls = ['from rhodecode.lib.dbmigrate.migrate.changeset import schema',
                  'pre_meta = MetaData()',
                  'post_meta = MetaData()',
                 ]
         upgradeCommands = ['pre_meta.bind = migrate_engine',
                            'post_meta.bind = migrate_engine']
         downgradeCommands = list(upgradeCommands)
 
         for tn in self.diff.tables_missing_from_A:
             pre_table = self.diff.metadataB.tables[tn]
             decls.extend(self._getTableDefn(pre_table, metaName='pre_meta'))
             upgradeCommands.append(
                 "pre_meta.tables[%(table)r].drop()" % {'table': tn})
             downgradeCommands.append(
                 "pre_meta.tables[%(table)r].create()" % {'table': tn})
 
         for tn in self.diff.tables_missing_from_B:
             post_table = self.diff.metadataA.tables[tn]
             decls.extend(self._getTableDefn(post_table, metaName='post_meta'))
             upgradeCommands.append(
                 "post_meta.tables[%(table)r].create()" % {'table': tn})
             downgradeCommands.append(
                 "post_meta.tables[%(table)r].drop()" % {'table': tn})
 
-        for (tn, td) in self.diff.tables_different.items():
+        for (tn, td) in list(self.diff.tables_different.items()):
             if td.columns_missing_from_A or td.columns_different:
                 pre_table = self.diff.metadataB.tables[tn]
                 decls.extend(self._getTableDefn(
                     pre_table, metaName='pre_meta'))
             if td.columns_missing_from_B or td.columns_different:
                 post_table = self.diff.metadataA.tables[tn]
                 decls.extend(self._getTableDefn(
                     post_table, metaName='post_meta'))
 
             for col in td.columns_missing_from_A:
                 upgradeCommands.append(
                     'pre_meta.tables[%r].columns[%r].drop()' % (tn, col))
                 downgradeCommands.append(
                     'pre_meta.tables[%r].columns[%r].create()' % (tn, col))
             for col in td.columns_missing_from_B:
                 upgradeCommands.append(
                     'post_meta.tables[%r].columns[%r].create()' % (tn, col))
                 downgradeCommands.append(
                     'post_meta.tables[%r].columns[%r].drop()' % (tn, col))
             for modelCol, databaseCol, modelDecl, databaseDecl in td.columns_different:
                 upgradeCommands.append(
                     'assert False, "Can\'t alter columns: %s:%s=>%s"' % (
                     tn, modelCol.name, databaseCol.name))
                 downgradeCommands.append(
                     'assert False, "Can\'t alter columns: %s:%s=>%s"' % (
                     tn, modelCol.name, databaseCol.name))
 
         return (
             '\n'.join(decls),
             '\n'.join('%s%s' % (indent, line) for line in upgradeCommands),
             '\n'.join('%s%s' % (indent, line) for line in downgradeCommands))
 
     def _db_can_handle_this_change(self,td):
         """Check if the database can handle going from B to A."""
 
         if (td.columns_missing_from_B
             and not td.columns_missing_from_A
             and not td.columns_different):
             # Even sqlite can handle column additions.
             return True
         else:
             return not self.engine.url.drivername.startswith('sqlite')
 
     def runB2A(self):
         """Goes from B to A.
 
         Was: applyModel. Apply model (A) to current database (B).
         """
 
         meta = sqlalchemy.MetaData(self.engine)
 
         for table in self._get_tables(missingA=True):
             table = table.tometadata(meta)
             table.drop()
         for table in self._get_tables(missingB=True):
             table = table.tometadata(meta)
             table.create()
         for modelTable in self._get_tables(modified=True):
             tableName = modelTable.name
             modelTable = modelTable.tometadata(meta)
             dbTable = self.diff.metadataB.tables[tableName]
 
             td = self.diff.tables_different[tableName]
 
             if self._db_can_handle_this_change(td):
 
                 for col in td.columns_missing_from_B:
                     modelTable.columns[col].create()
                 for col in td.columns_missing_from_A:
                     dbTable.columns[col].drop()
                 # XXX handle column changes here.
             else:
                 # Sqlite doesn't support drop column, so you have to
                 # do more: create temp table, copy data to it, drop
                 # old table, create new table, copy data back.
                 #
                 # I wonder if this is guaranteed to be unique?
                 tempName = '_temp_%s' % modelTable.name
 
                 def getCopyStatement():
                     preparer = self.engine.dialect.preparer
                     commonCols = []
                     for modelCol in modelTable.columns:
                         if modelCol.name in dbTable.columns:
                             commonCols.append(modelCol.name)
                     commonColsStr = ', '.join(commonCols)
                     return 'INSERT INTO %s (%s) SELECT %s FROM %s' % \
                         (tableName, commonColsStr, commonColsStr, tempName)
 
                 # Move the data in one transaction, so that we don't
                 # leave the database in a nasty state.
                 connection = self.engine.connect()
                 trans = connection.begin()
                 try:
                     connection.execute(
                         'CREATE TEMPORARY TABLE %s as SELECT * from %s' % \
                             (tempName, modelTable.name))
                     # make sure the drop takes place inside our
                     # transaction with the bind parameter
                     modelTable.drop(bind=connection)
                     modelTable.create(bind=connection)
                     connection.execute(getCopyStatement())
                     connection.execute('DROP TABLE %s' % tempName)
                     trans.commit()
                 except:
                     trans.rollback()
                     raise

rhodecode/lib/dbmigrate/migrate/versioning/migrate_repository.py

@@ -1,100 +1,100 b''
 """
    Script to migrate repository from sqlalchemy <= 0.4.4 to the new
    repository schema. This shouldn't use any other migrate modules, so
    that it can work in any version.
 """
 
 import os
 import sys
 import logging
 
 log = logging.getLogger(__name__)
 
 
 def usage():
     """Gives usage information."""
-    print("""Usage: %(prog)s repository-to-migrate
+    print(("""Usage: %(prog)s repository-to-migrate
 
     Upgrade your repository to the new flat format.
 
     NOTE: You should probably make a backup before running this.
-    """ % {'prog': sys.argv[0]})
+    """ % {'prog': sys.argv[0]}))
 
     sys.exit(1)
 
 
 def delete_file(filepath):
     """Deletes a file and prints a message."""
     log.info('Deleting file: %s', filepath)
     os.remove(filepath)
 
 
 def move_file(src, tgt):
     """Moves a file and prints a message."""
     log.info('Moving file %s to %s', src, tgt)
     if os.path.exists(tgt):
         raise Exception(
             'Cannot move file %s because target %s already exists' % \
                 (src, tgt))
     os.rename(src, tgt)
 
 
 def delete_directory(dirpath):
     """Delete a directory and print a message."""
     log.info('Deleting directory: %s', dirpath)
     os.rmdir(dirpath)
 
 
 def migrate_repository(repos):
     """Does the actual migration to the new repository format."""
     log.info('Migrating repository at: %s to new format', repos)
     versions = '%s/versions' % repos
     dirs = os.listdir(versions)
     # Only use int's in list.
     numdirs = [int(dirname) for dirname in dirs if dirname.isdigit()]
     numdirs.sort()  # Sort list.
     for dirname in numdirs:
         origdir = '%s/%s' % (versions, dirname)
         log.info('Working on directory: %s', origdir)
         files = os.listdir(origdir)
         files.sort()
         for filename in files:
             # Delete compiled Python files.
             if filename.endswith('.pyc') or filename.endswith('.pyo'):
                 delete_file('%s/%s' % (origdir, filename))
 
             # Delete empty __init__.py files.
             origfile = '%s/__init__.py' % origdir
             if os.path.exists(origfile) and len(open(origfile).read()) == 0:
                 delete_file(origfile)
 
             # Move sql upgrade scripts.
             if filename.endswith('.sql'):
                 version, dbms, operation = filename.split('.', 3)[0:3]
                 origfile = '%s/%s' % (origdir, filename)
                 # For instance:  2.postgres.upgrade.sql ->
                 #  002_postgres_upgrade.sql
                 tgtfile = '%s/%03d_%s_%s.sql' % (
                     versions, int(version), dbms, operation)
                 move_file(origfile, tgtfile)
 
         # Move Python upgrade script.
         pyfile = '%s.py' % dirname
         pyfilepath = '%s/%s' % (origdir, pyfile)
         if os.path.exists(pyfilepath):
             tgtfile = '%s/%03d.py' % (versions, int(dirname))
             move_file(pyfilepath, tgtfile)
 
         # Try to remove directory. Will fail if it's not empty.
         delete_directory(origdir)
 
 
 def main():
     """Main function to be called when using this script."""
     if len(sys.argv) != 2:
         usage()
     migrate_repository(sys.argv[1])
 
 
 if __name__ == '__main__':
     main()

rhodecode/lib/dbmigrate/migrate/versioning/repository.py

@@ -1,243 +1,243 b''
 """
    SQLAlchemy migrate repository management.
 """
 import os
 import shutil
 import string
 import logging
 
 from pkg_resources import resource_filename
 from tempita import Template as TempitaTemplate
 
 from rhodecode.lib.dbmigrate.migrate import exceptions
 from rhodecode.lib.dbmigrate.migrate.versioning import version, pathed, cfgparse
 from rhodecode.lib.dbmigrate.migrate.versioning.template import Template
 from rhodecode.lib.dbmigrate.migrate.versioning.config import *
 
 
 log = logging.getLogger(__name__)
 
 class Changeset(dict):
     """A collection of changes to be applied to a database.
 
     Changesets are bound to a repository and manage a set of
     scripts from that repository.
 
     Behaves like a dict, for the most part. Keys are ordered based on step value.
     """
 
     def __init__(self, start, *changes, **k):
         """
         Give a start version; step must be explicitly stated.
         """
         self.step = k.pop('step', 1)
         self.start = version.VerNum(start)
         self.end = self.start
         for change in changes:
             self.add(change)
 
     def __iter__(self):
-        return iter(self.items())
+        return iter(list(self.items()))
 
     def keys(self):
         """
         In a series of upgrades x -> y, keys are version x. Sorted.
         """
-        ret = super(Changeset, self).keys()
+        ret = list(super(Changeset, self).keys())
         # Reverse order if downgrading
         ret.sort(reverse=(self.step < 1))
         return ret
 
     def values(self):
-        return [self[k] for k in self.keys()]
+        return [self[k] for k in list(self.keys())]
 
     def items(self):
-        return zip(self.keys(), self.values())
+        return list(zip(list(self.keys()), list(self.values())))
 
     def add(self, change):
         """Add new change to changeset"""
         key = self.end
         self.end += self.step
         self[key] = change
 
     def run(self, *p, **k):
         """Run the changeset scripts"""
         for version, script in self:
             script.run(*p, **k)
 
 
 class Repository(pathed.Pathed):
     """A project's change script repository"""
 
     _config = 'migrate.cfg'
     _versions = 'versions'
 
     def __init__(self, path):
         log.debug('Loading repository %s...', path)
         self.verify(path)
         super(Repository, self).__init__(path)
         self.config = cfgparse.Config(os.path.join(self.path, self._config))
         self.versions = version.Collection(os.path.join(self.path,
                                                       self._versions))
         log.debug('Repository %s loaded successfully', path)
         log.debug('Config: %r', self.config.to_dict())
 
     @classmethod
     def verify(cls, path):
         """
         Ensure the target path is a valid repository.
 
         :raises: :exc:`InvalidRepositoryError <migrate.exceptions.InvalidRepositoryError>`
         """
         # Ensure the existence of required files
         try:
             cls.require_found(path)
             cls.require_found(os.path.join(path, cls._config))
             cls.require_found(os.path.join(path, cls._versions))
         except exceptions.PathNotFoundError as e:
             raise exceptions.InvalidRepositoryError(path)
 
     @classmethod
     def prepare_config(cls, tmpl_dir, name, options=None):
         """
         Prepare a project configuration file for a new project.
 
         :param tmpl_dir: Path to Repository template
         :param config_file: Name of the config file in Repository template
         :param name: Repository name
         :type tmpl_dir: string
         :type config_file: string
         :type name: string
         :returns: Populated config file
         """
         if options is None:
             options = {}
         options.setdefault('version_table', 'migrate_version')
         options.setdefault('repository_id', name)
         options.setdefault('required_dbs', [])
         options.setdefault('use_timestamp_numbering', False)
 
         with open(os.path.join(tmpl_dir, cls._config)) as f:
             tmpl = f.read()
         ret = TempitaTemplate(tmpl).substitute(options)
 
         # cleanup
         del options['__template_name__']
 
         return ret
 
     @classmethod
     def create(cls, path, name, **opts):
         """Create a repository at a specified path"""
         cls.require_notfound(path)
         theme = opts.pop('templates_theme', None)
         t_path = opts.pop('templates_path', None)
 
         # Create repository
         tmpl_dir = Template(t_path).get_repository(theme=theme)
         shutil.copytree(tmpl_dir, path)
 
         # Edit config defaults
         config_text = cls.prepare_config(tmpl_dir, name, options=opts)
         with open(os.path.join(path, cls._config), 'w') as fd:
             fd.write(config_text)
 
         opts['repository_name'] = name
 
         # Create a management script
         manager = os.path.join(path, 'manage.py')
         Repository.create_manage_file(manager, templates_theme=theme,
             templates_path=t_path, **opts)
 
         return cls(path)
 
     def create_script(self, description, **k):
         """API to :meth:`migrate.versioning.version.Collection.create_new_python_version`"""
 
         k['use_timestamp_numbering'] = self.use_timestamp_numbering
         self.versions.create_new_python_version(description, **k)
 
     def create_script_sql(self, database, description, **k):
         """API to :meth:`migrate.versioning.version.Collection.create_new_sql_version`"""
         k['use_timestamp_numbering'] = self.use_timestamp_numbering
         self.versions.create_new_sql_version(database, description, **k)
 
     @property
     def latest(self):
         """API to :attr:`migrate.versioning.version.Collection.latest`"""
         return self.versions.latest
 
     @property
     def version_table(self):
         """Returns version_table name specified in config"""
         return self.config.get('db_settings', 'version_table')
 
     @property
     def id(self):
         """Returns repository id specified in config"""
         return self.config.get('db_settings', 'repository_id')
 
     @property
     def use_timestamp_numbering(self):
         """Returns use_timestamp_numbering specified in config"""
         if self.config.has_option('db_settings', 'use_timestamp_numbering'):
             return self.config.getboolean('db_settings', 'use_timestamp_numbering')
         return False
 
     def version(self, *p, **k):
         """API to :attr:`migrate.versioning.version.Collection.version`"""
         return self.versions.version(*p, **k)
 
     @classmethod
     def clear(cls):
         # TODO: deletes repo
         super(Repository, cls).clear()
         version.Collection.clear()
 
     def changeset(self, database, start, end=None):
         """Create a changeset to migrate this database from ver. start to end/latest.
 
         :param database: name of database to generate changeset
         :param start: version to start at
         :param end: version to end at (latest if None given)
         :type database: string
         :type start: int
         :type end: int
         :returns: :class:`Changeset instance <migration.versioning.repository.Changeset>`
         """
         start = version.VerNum(start)
 
         if end is None:
             end = self.latest
         else:
             end = version.VerNum(end)
 
         if start <= end:
             step = 1
             range_mod = 1
             op = 'upgrade'
         else:
             step = -1
             range_mod = 0
             op = 'downgrade'
 
-        versions = range(int(start) + range_mod, int(end) + range_mod, step)
+        versions = list(range(int(start) + range_mod, int(end) + range_mod, step))
         changes = [self.version(v).script(database, op) for v in versions]
         ret = Changeset(start, step=step, *changes)
         return ret
 
     @classmethod
     def create_manage_file(cls, file_, **opts):
         """Create a project management script (manage.py)
 
         :param file_: Destination file to be written
         :param opts: Options that are passed to :func:`migrate.versioning.shell.main`
         """
         mng_file = Template(opts.pop('templates_path', None))\
             .get_manage(theme=opts.pop('templates_theme', None))
 
         with open(mng_file) as f:
             tmpl = f.read()
 
         with open(file_, 'w') as fd:
             fd.write(TempitaTemplate(tmpl).substitute(opts))

rhodecode/lib/dbmigrate/migrate/versioning/schema.py

@@ -1,221 +1,221 b''
 """
    Database schema version management.
 """
 import sys
 import logging
 
 from sqlalchemy import (Table, Column, MetaData, String, Text, Integer,
     create_engine)
 from sqlalchemy.sql import and_
 from sqlalchemy import exc as sa_exceptions
 from sqlalchemy.sql import bindparam
 
 from rhodecode.lib.dbmigrate.migrate import exceptions
 from rhodecode.lib.dbmigrate.migrate.changeset import SQLA_07
 from rhodecode.lib.dbmigrate.migrate.versioning import genmodel, schemadiff
 from rhodecode.lib.dbmigrate.migrate.versioning.repository import Repository
 from rhodecode.lib.dbmigrate.migrate.versioning.util import load_model
 from rhodecode.lib.dbmigrate.migrate.versioning.version import VerNum
 
 
 log = logging.getLogger(__name__)
 
 
 class ControlledSchema(object):
     """A database under version control"""
 
     def __init__(self, engine, repository):
         if isinstance(repository, str):
             repository = Repository(repository)
         self.engine = engine
         self.repository = repository
         self.meta = MetaData(engine)
         self.load()
 
     def __eq__(self, other):
         """Compare two schemas by repositories and versions"""
         return (self.repository is other.repository \
             and self.version == other.version)
 
     def load(self):
         """Load controlled schema version info from DB"""
         tname = self.repository.version_table
         try:
             if not hasattr(self, 'table') or self.table is None:
                     self.table = Table(tname, self.meta, autoload=True)
 
             result = self.engine.execute(self.table.select(
                 self.table.c.repository_id == str(self.repository.id)))
 
             data = list(result)[0]
         except:
             cls, exc, tb = sys.exc_info()
-            raise exceptions.DatabaseNotControlledError, exc.__str__(), tb
+            raise exceptions.DatabaseNotControlledError(exc.__str__()).with_traceback(tb)
 
         self.version = data['version']
         return data
 
     def drop(self):
         """
         Remove version control from a database.
         """
         if SQLA_07:
             try:
                 self.table.drop()
             except sa_exceptions.DatabaseError:
                 raise exceptions.DatabaseNotControlledError(str(self.table))
         else:
             try:
                 self.table.drop()
             except (sa_exceptions.SQLError):
                 raise exceptions.DatabaseNotControlledError(str(self.table))
 
     def changeset(self, version=None):
         """API to Changeset creation.
 
         Uses self.version for start version and engine.name
         to get database name.
         """
         database = self.engine.name
         start_ver = self.version
         changeset = self.repository.changeset(database, start_ver, version)
         return changeset
 
     def runchange(self, ver, change, step):
         startver = ver
         endver = ver + step
         # Current database version must be correct! Don't run if corrupt!
         if self.version != startver:
             raise exceptions.InvalidVersionError("%s is not %s" % \
                                                      (self.version, startver))
         # Run the change
         change.run(self.engine, step)
 
         # Update/refresh database version
         self.update_repository_table(startver, endver)
         self.load()
 
     def update_repository_table(self, startver, endver):
         """Update version_table with new information"""
         update = self.table.update(and_(self.table.c.version == int(startver),
              self.table.c.repository_id == str(self.repository.id)))
         self.engine.execute(update, version=int(endver))
 
     def upgrade(self, version=None):
         """
         Upgrade (or downgrade) to a specified version, or latest version.
         """
         changeset = self.changeset(version)
         for ver, change in changeset:
             self.runchange(ver, change, changeset.step)
 
     def update_db_from_model(self, model):
         """
         Modify the database to match the structure of the current Python model.
         """
         model = load_model(model)
 
         diff = schemadiff.getDiffOfModelAgainstDatabase(
             model, self.engine, excludeTables=[self.repository.version_table]
             )
         genmodel.ModelGenerator(diff,self.engine).runB2A()
 
         self.update_repository_table(self.version, int(self.repository.latest))
 
         self.load()
 
     @classmethod
     def create(cls, engine, repository, version=None):
         """
         Declare a database to be under a repository's version control.
 
         :raises: :exc:`DatabaseAlreadyControlledError`
         :returns: :class:`ControlledSchema`
         """
         # Confirm that the version # is valid: positive, integer,
         # exists in repos
         if isinstance(repository, str):
             repository = Repository(repository)
         version = cls._validate_version(repository, version)
         table = cls._create_table_version(engine, repository, version)
         # TODO: history table
         # Load repository information and return
         return cls(engine, repository)
 
     @classmethod
     def _validate_version(cls, repository, version):
         """
         Ensures this is a valid version number for this repository.
 
         :raises: :exc:`InvalidVersionError` if invalid
         :return: valid version number
         """
         if version is None:
             version = 0
         try:
             version = VerNum(version) # raises valueerror
             if version < 0 or version > repository.latest:
                 raise ValueError()
         except ValueError:
             raise exceptions.InvalidVersionError(version)
         return version
 
     @classmethod
     def _create_table_version(cls, engine, repository, version):
         """
         Creates the versioning table in a database.
 
         :raises: :exc:`DatabaseAlreadyControlledError`
         """
         # Create tables
         tname = repository.version_table
         meta = MetaData(engine)
 
         table = Table(
             tname, meta,
             Column('repository_id', String(250), primary_key=True),
             Column('repository_path', Text),
             Column('version', Integer), )
 
         # there can be multiple repositories/schemas in the same db
         if not table.exists():
             table.create()
 
         # test for existing repository_id
         s = table.select(table.c.repository_id == bindparam("repository_id"))
         result = engine.execute(s, repository_id=repository.id)
         if result.fetchone():
             raise exceptions.DatabaseAlreadyControlledError
 
         # Insert data
         engine.execute(table.insert().values(
                            repository_id=repository.id,
                            repository_path=repository.path,
                            version=int(version)))
         return table
 
     @classmethod
     def compare_model_to_db(cls, engine, model, repository):
         """
         Compare the current model against the current database.
         """
         if isinstance(repository, str):
             repository = Repository(repository)
         model = load_model(model)
 
         diff = schemadiff.getDiffOfModelAgainstDatabase(
             model, engine, excludeTables=[repository.version_table])
         return diff
 
     @classmethod
     def create_model(cls, engine, repository, declarative=False):
         """
         Dump the current database as a Python model.
         """
         if isinstance(repository, str):
             repository = Repository(repository)
 
         diff = schemadiff.getDiffOfModelAgainstDatabase(
             MetaData(), engine, excludeTables=[repository.version_table]
             )
         return genmodel.ModelGenerator(diff, engine, declarative).genBDefinition()

rhodecode/lib/dbmigrate/migrate/versioning/schemadiff.py

@@ -1,299 +1,299 b''
 """
    Schema differencing support.
 """
 
 import logging
 import sqlalchemy
 
 from sqlalchemy.types import Float
 
 log = logging.getLogger(__name__)
 
 
 def getDiffOfModelAgainstDatabase(metadata, engine, excludeTables=None):
     """
     Return differences of model against database.
 
     :return: object which will evaluate to :keyword:`True` if there \
       are differences else :keyword:`False`.
     """
     db_metadata = sqlalchemy.MetaData(engine)
     db_metadata.reflect()
 
     # sqlite will include a dynamically generated 'sqlite_sequence' table if
     # there are autoincrement sequences in the database; this should not be
     # compared.
     if engine.dialect.name == 'sqlite':
         if 'sqlite_sequence' in db_metadata.tables:
             db_metadata.remove(db_metadata.tables['sqlite_sequence'])
 
     return SchemaDiff(metadata, db_metadata,
                       labelA='model',
                       labelB='database',
                       excludeTables=excludeTables)
 
 
 def getDiffOfModelAgainstModel(metadataA, metadataB, excludeTables=None):
     """
     Return differences of model against another model.
 
     :return: object which will evaluate to :keyword:`True` if there \
       are differences else :keyword:`False`.
     """
     return SchemaDiff(metadataA, metadataB, excludeTables=excludeTables)
 
 
 class ColDiff(object):
     """
     Container for differences in one :class:`~sqlalchemy.schema.Column`
     between two :class:`~sqlalchemy.schema.Table` instances, ``A``
     and ``B``.
 
     .. attribute:: col_A
 
       The :class:`~sqlalchemy.schema.Column` object for A.
 
     .. attribute:: col_B
 
       The :class:`~sqlalchemy.schema.Column` object for B.
 
     .. attribute:: type_A
 
       The most generic type of the :class:`~sqlalchemy.schema.Column`
       object in A.
 
     .. attribute:: type_B
 
       The most generic type of the :class:`~sqlalchemy.schema.Column`
       object in A.
 
     """
 
     diff = False
 
     def __init__(self,col_A,col_B):
         self.col_A = col_A
         self.col_B = col_B
 
         self.type_A = col_A.type
         self.type_B = col_B.type
 
         self.affinity_A = self.type_A._type_affinity
         self.affinity_B = self.type_B._type_affinity
 
         if self.affinity_A is not self.affinity_B:
             self.diff = True
             return
 
         if isinstance(self.type_A,Float) or isinstance(self.type_B,Float):
             if not (isinstance(self.type_A,Float) and isinstance(self.type_B,Float)):
                 self.diff=True
                 return
 
         for attr in ('precision','scale','length'):
             A = getattr(self.type_A,attr,None)
             B = getattr(self.type_B,attr,None)
             if not (A is None or B is None) and A!=B:
                 self.diff=True
                 return
 
-    def __nonzero__(self):
+    def __bool__(self):
         return self.diff
 
     __bool__ = __nonzero__
 
 
 class TableDiff(object):
     """
     Container for differences in one :class:`~sqlalchemy.schema.Table`
     between two :class:`~sqlalchemy.schema.MetaData` instances, ``A``
     and ``B``.
 
     .. attribute:: columns_missing_from_A
 
       A sequence of column names that were found in B but weren't in
       A.
 
     .. attribute:: columns_missing_from_B
 
       A sequence of column names that were found in A but weren't in
       B.
 
     .. attribute:: columns_different
 
       A dictionary containing information about columns that were
       found to be different.
       It maps column names to a :class:`ColDiff` objects describing the
       differences found.
     """
     __slots__ = (
         'columns_missing_from_A',
         'columns_missing_from_B',
         'columns_different',
         )
 
-    def __nonzero__(self):
+    def __bool__(self):
         return bool(
             self.columns_missing_from_A or
             self.columns_missing_from_B or
             self.columns_different
             )
 
     __bool__ = __nonzero__
 
 class SchemaDiff(object):
     """
     Compute the difference between two :class:`~sqlalchemy.schema.MetaData`
     objects.
 
     The string representation of a :class:`SchemaDiff` will summarise
     the changes found between the two
     :class:`~sqlalchemy.schema.MetaData` objects.
 
     The length of a :class:`SchemaDiff` will give the number of
     changes found, enabling it to be used much like a boolean in
     expressions.
 
     :param metadataA:
       First :class:`~sqlalchemy.schema.MetaData` to compare.
 
     :param metadataB:
       Second :class:`~sqlalchemy.schema.MetaData` to compare.
 
     :param labelA:
       The label to use in messages about the first
       :class:`~sqlalchemy.schema.MetaData`.
 
     :param labelB:
       The label to use in messages about the second
       :class:`~sqlalchemy.schema.MetaData`.
 
     :param excludeTables:
       A sequence of table names to exclude.
 
     .. attribute:: tables_missing_from_A
 
       A sequence of table names that were found in B but weren't in
       A.
 
     .. attribute:: tables_missing_from_B
 
       A sequence of table names that were found in A but weren't in
       B.
 
     .. attribute:: tables_different
 
       A dictionary containing information about tables that were found
       to be different.
       It maps table names to a :class:`TableDiff` objects describing the
       differences found.
     """
 
     def __init__(self,
                  metadataA, metadataB,
                  labelA='metadataA',
                  labelB='metadataB',
                  excludeTables=None):
 
         self.metadataA, self.metadataB = metadataA, metadataB
         self.labelA, self.labelB = labelA, labelB
         self.label_width = max(len(labelA),len(labelB))
         excludeTables = set(excludeTables or [])
 
         A_table_names = set(metadataA.tables.keys())
         B_table_names = set(metadataB.tables.keys())
 
         self.tables_missing_from_A = sorted(
             B_table_names - A_table_names - excludeTables
             )
         self.tables_missing_from_B = sorted(
             A_table_names - B_table_names - excludeTables
             )
 
         self.tables_different = {}
         for table_name in A_table_names.intersection(B_table_names):
 
             td = TableDiff()
 
             A_table = metadataA.tables[table_name]
             B_table = metadataB.tables[table_name]
 
             A_column_names = set(A_table.columns.keys())
             B_column_names = set(B_table.columns.keys())
 
             td.columns_missing_from_A = sorted(
                 B_column_names - A_column_names
                 )
 
             td.columns_missing_from_B = sorted(
                 A_column_names - B_column_names
                 )
 
             td.columns_different = {}
 
             for col_name in A_column_names.intersection(B_column_names):
 
                 cd = ColDiff(
                     A_table.columns.get(col_name),
                     B_table.columns.get(col_name)
                     )
 
                 if cd:
                     td.columns_different[col_name]=cd
 
             # XXX - index and constraint differences should
             #       be checked for here
 
             if td:
                 self.tables_different[table_name]=td
 
     def __str__(self):
         """ Summarize differences. """
         out = []
         column_template ='      %%%is: %%r' % self.label_width
 
         for names,label in (
             (self.tables_missing_from_A,self.labelA),
             (self.tables_missing_from_B,self.labelB),
             ):
             if names:
                 out.append(
                     '  tables missing from %s: %s' % (
                         label,', '.join(sorted(names))
                         )
                     )
 
         for name,td in sorted(self.tables_different.items()):
             out.append(
                '  table with differences: %s' % name
                )
             for names,label in (
                 (td.columns_missing_from_A,self.labelA),
                 (td.columns_missing_from_B,self.labelB),
                 ):
                 if names:
                     out.append(
                         '    %s missing these columns: %s' % (
                             label,', '.join(sorted(names))
                             )
                         )
-            for name,cd in td.columns_different.items():
+            for name,cd in list(td.columns_different.items()):
                 out.append('    column with differences: %s' % name)
                 out.append(column_template % (self.labelA,cd.col_A))
                 out.append(column_template % (self.labelB,cd.col_B))
 
         if out:
             out.insert(0, 'Schema diffs:')
             return '\n'.join(out)
         else:
             return 'No schema diffs'
 
     def __len__(self):
         """
         Used in bool evaluation, return of 0 means no diffs.
         """
         return (
             len(self.tables_missing_from_A) +
             len(self.tables_missing_from_B) +
             len(self.tables_different)
             )

rhodecode/lib/dbmigrate/migrate/versioning/shell.py

@@ -1,215 +1,215 b''
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 """The migrate command-line tool."""
 
 import sys
 import inspect
 import logging
 from optparse import OptionParser, BadOptionError
 
 from rhodecode.lib.dbmigrate.migrate import exceptions
 from rhodecode.lib.dbmigrate.migrate.versioning import api
 from rhodecode.lib.dbmigrate.migrate.versioning.config import *
 from rhodecode.lib.dbmigrate.migrate.versioning.util import asbool
 
 
 alias = {
     's': api.script,
     'vc': api.version_control,
     'dbv': api.db_version,
     'v': api.version,
 }
 
 def alias_setup():
     global alias
-    for key, val in alias.items():
+    for key, val in list(alias.items()):
         setattr(api, key, val)
 alias_setup()
 
 
 class PassiveOptionParser(OptionParser):
 
     def _process_args(self, largs, rargs, values):
         """little hack to support all --some_option=value parameters"""
 
         while rargs:
             arg = rargs[0]
             if arg == "--":
                 del rargs[0]
                 return
             elif arg[0:2] == "--":
                 # if parser does not know about the option
                 # pass it along (make it anonymous)
                 try:
                     opt = arg.split('=', 1)[0]
                     self._match_long_opt(opt)
                 except BadOptionError:
                     largs.append(arg)
                     del rargs[0]
                 else:
                     self._process_long_opt(rargs, values)
             elif arg[:1] == "-" and len(arg) > 1:
                 self._process_short_opts(rargs, values)
             elif self.allow_interspersed_args:
                 largs.append(arg)
                 del rargs[0]
 
 def main(argv=None, **kwargs):
     """Shell interface to :mod:`migrate.versioning.api`.
 
     kwargs are default options that can be overriden with passing
     --some_option as command line option
 
     :param disable_logging: Let migrate configure logging
     :type disable_logging: bool
     """
     if argv is not None:
         argv = argv
     else:
         argv = list(sys.argv[1:])
     commands = list(api.__all__)
     commands.sort()
 
     usage = """%%prog COMMAND ...
 
     Available commands:
         %s
 
     Enter "%%prog help COMMAND" for information on a particular command.
     """ % '\n\t'.join(["%s - %s" % (command.ljust(28), api.command_desc.get(command)) for command in commands])
 
     parser = PassiveOptionParser(usage=usage)
     parser.add_option("-d", "--debug",
                      action="store_true",
                      dest="debug",
                      default=False,
                      help="Shortcut to turn on DEBUG mode for logging")
     parser.add_option("-q", "--disable_logging",
                       action="store_true",
                       dest="disable_logging",
                       default=False,
                       help="Use this option to disable logging configuration")
     help_commands = ['help', '-h', '--help']
     HELP = False
 
     try:
         command = argv.pop(0)
         if command in help_commands:
             HELP = True
             command = argv.pop(0)
     except IndexError:
         parser.print_help()
         return
 
     command_func = getattr(api, command, None)
     if command_func is None or command.startswith('_'):
         parser.error("Invalid command %s" % command)
 
     parser.set_usage(inspect.getdoc(command_func))
     f_args, f_varargs, f_kwargs, f_defaults = inspect.getargspec(command_func)
     for arg in f_args:
         parser.add_option(
             "--%s" % arg,
             dest=arg,
             action='store',
             type="string")
 
     # display help of the current command
     if HELP:
         parser.print_help()
         return
 
     options, args = parser.parse_args(argv)
 
     # override kwargs with anonymous parameters
     override_kwargs = {}
     for arg in list(args):
         if arg.startswith('--'):
             args.remove(arg)
             if '=' in arg:
                 opt, value = arg[2:].split('=', 1)
             else:
                 opt = arg[2:]
                 value = True
             override_kwargs[opt] = value
 
     # override kwargs with options if user is overwriting
-    for key, value in options.__dict__.items():
+    for key, value in list(options.__dict__.items()):
         if value is not None:
             override_kwargs[key] = value
 
     # arguments that function accepts without passed kwargs
     f_required = list(f_args)
     candidates = dict(kwargs)
     candidates.update(override_kwargs)
-    for key, value in candidates.items():
+    for key, value in list(candidates.items()):
         if key in f_args:
             f_required.remove(key)
 
     # map function arguments to parsed arguments
     for arg in args:
         try:
             kw = f_required.pop(0)
         except IndexError:
             parser.error("Too many arguments for command %s: %s" % (command,
                                                                     arg))
         kwargs[kw] = arg
 
     # apply overrides
     kwargs.update(override_kwargs)
 
     # configure options
-    for key, value in options.__dict__.items():
+    for key, value in list(options.__dict__.items()):
         kwargs.setdefault(key, value)
 
     # configure logging
     if not asbool(kwargs.pop('disable_logging', False)):
         # filter to log =< INFO into stdout and rest to stderr
         class SingleLevelFilter(logging.Filter):
             def __init__(self, min=None, max=None):
                 self.min = min or 0
                 self.max = max or 100
 
             def filter(self, record):
                 return self.min <= record.levelno <= self.max
 
         logger = logging.getLogger()
         h1 = logging.StreamHandler(sys.stdout)
         f1 = SingleLevelFilter(max=logging.INFO)
         h1.addFilter(f1)
         h2 = logging.StreamHandler(sys.stderr)
         f2 = SingleLevelFilter(min=logging.WARN)
         h2.addFilter(f2)
         logger.addHandler(h1)
         logger.addHandler(h2)
 
         if options.debug:
             logger.setLevel(logging.DEBUG)
         else:
             logger.setLevel(logging.INFO)
 
     log = logging.getLogger(__name__)
 
     # check if all args are given
     try:
         num_defaults = len(f_defaults)
     except TypeError:
         num_defaults = 0
     f_args_default = f_args[len(f_args) - num_defaults:]
     required = list(set(f_required) - set(f_args_default))
     required.sort()
     if required:
         parser.error("Not enough arguments for command %s: %s not specified" \
             % (command, ', '.join(required)))
 
     # handle command
     try:
         ret = command_func(**kwargs)
         if ret is not None:
             log.info(ret)
     except (exceptions.UsageError, exceptions.KnownError) as e:
         parser.error(e.args[0])
 
 if __name__ == "__main__":
     main()

rhodecode/lib/dbmigrate/migrate/versioning/util/__init__.py

@@ -1,180 +1,180 b''
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 """.. currentmodule:: migrate.versioning.util"""
 
 import warnings
 import logging
 from decorator import decorator
 from pkg_resources import EntryPoint
 
 from sqlalchemy import create_engine
 from sqlalchemy.engine import Engine
 from sqlalchemy.pool import StaticPool
 
 from rhodecode.lib.dbmigrate.migrate import exceptions
 from rhodecode.lib.dbmigrate.migrate.versioning.util.keyedinstance import KeyedInstance
 from rhodecode.lib.dbmigrate.migrate.versioning.util.importpath import import_path
 
 
 log = logging.getLogger(__name__)
 
 
 def load_model(dotted_name):
     """Import module and use module-level variable".
 
     :param dotted_name: path to model in form of string: ``some.python.module:Class``
 
     .. versionchanged:: 0.5.4
 
     """
     if isinstance(dotted_name, str):
         if ':' not in dotted_name:
             # backwards compatibility
             warnings.warn('model should be in form of module.model:User '
                 'and not module.model.User', exceptions.MigrateDeprecationWarning)
             dotted_name = ':'.join(dotted_name.rsplit('.', 1))
         return EntryPoint.parse('x=%s' % dotted_name).load(False)
     else:
         # Assume it's already loaded.
         return dotted_name
 
 def asbool(obj):
     """Do everything to use object as bool"""
     if isinstance(obj, str):
         obj = obj.strip().lower()
         if obj in ['true', 'yes', 'on', 'y', 't', '1']:
             return True
         elif obj in ['false', 'no', 'off', 'n', 'f', '0']:
             return False
         else:
             raise ValueError("String is not true/false: %r" % obj)
     if obj in (True, False):
         return bool(obj)
     else:
         raise ValueError("String is not true/false: %r" % obj)
 
 def guess_obj_type(obj):
     """Do everything to guess object type from string
 
     Tries to convert to `int`, `bool` and finally returns if not succeded.
 
     .. versionadded: 0.5.4
     """
 
     result = None
 
     try:
         result = int(obj)
     except:
         pass
 
     if result is None:
         try:
             result = asbool(obj)
         except:
             pass
 
     if result is not None:
         return result
     else:
         return obj
 
 @decorator
 def catch_known_errors(f, *a, **kw):
     """Decorator that catches known api errors
 
     .. versionadded: 0.5.4
     """
 
     try:
         return f(*a, **kw)
     except exceptions.PathFoundError as e:
         raise exceptions.KnownError("The path %s already exists" % e.args[0])
 
 def construct_engine(engine, **opts):
     """.. versionadded:: 0.5.4
 
     Constructs and returns SQLAlchemy engine.
 
     Currently, there are 2 ways to pass create_engine options to :mod:`migrate.versioning.api` functions:
 
     :param engine: connection string or a existing engine
     :param engine_dict: python dictionary of options to pass to `create_engine`
     :param engine_arg_*: keyword parameters to pass to `create_engine` (evaluated with :func:`migrate.versioning.util.guess_obj_type`)
     :type engine_dict: dict
     :type engine: string or Engine instance
     :type engine_arg_*: string
     :returns: SQLAlchemy Engine
 
     .. note::
 
         keyword parameters override ``engine_dict`` values.
 
     """
     if isinstance(engine, Engine):
         return engine
     elif not isinstance(engine, str):
         raise ValueError("you need to pass either an existing engine or a database uri")
 
     # get options for create_engine
     if opts.get('engine_dict') and isinstance(opts['engine_dict'], dict):
         kwargs = opts['engine_dict']
     else:
         kwargs = {}
 
     # DEPRECATED: handle echo the old way
     echo = asbool(opts.get('echo', False))
     if echo:
         warnings.warn('echo=True parameter is deprecated, pass '
             'engine_arg_echo=True or engine_dict={"echo": True}',
             exceptions.MigrateDeprecationWarning)
         kwargs['echo'] = echo
 
     # parse keyword arguments
-    for key, value in opts.items():
+    for key, value in list(opts.items()):
         if key.startswith('engine_arg_'):
             kwargs[key[11:]] = guess_obj_type(value)
 
     log.debug('Constructing engine')
     # TODO: return create_engine(engine, poolclass=StaticPool, **kwargs)
     # seems like 0.5.x branch does not work with engine.dispose and staticpool
     return create_engine(engine, **kwargs)
 
 @decorator
 def with_engine(f, *a, **kw):
     """Decorator for :mod:`migrate.versioning.api` functions
     to safely close resources after function usage.
 
     Passes engine parameters to :func:`construct_engine` and
     resulting parameter is available as kw['engine'].
 
     Engine is disposed after wrapped function is executed.
 
     .. versionadded: 0.6.0
     """
     url = a[0]
     engine = construct_engine(url, **kw)
 
     try:
         kw['engine'] = engine
         return f(*a, **kw)
     finally:
         if isinstance(engine, Engine) and engine is not url:
             log.debug('Disposing SQLAlchemy engine %s', engine)
             engine.dispose()
 
 
 class Memoize:
     """Memoize(fn) - an instance which acts like fn but memoizes its arguments
        Will only work on functions with non-mutable arguments
 
        ActiveState Code 52201
     """
     def __init__(self, fn):
         self.fn = fn
         self.memo = {}
 
     def __call__(self, *args):
         if args not in self.memo:
             self.memo[args] = self.fn(*args)
         return self.memo[args]

rhodecode/lib/dbmigrate/migrate/versioning/util/importpath.py

@@ -1,15 +1,16 b''
 import os
 import sys
+import importlib
 
 def import_path(fullpath):
     """ Import a file with full path specification. Allows one to
         import from anywhere, something __import__ does not do.
     """
     # http://zephyrfalcon.org/weblog/arch_d7_2002_08_31.html
     path, filename = os.path.split(fullpath)
     filename, ext = os.path.splitext(filename)
     sys.path.append(path)
     module = __import__(filename)
-    reload(module) # Might be out of date during tests
+    importlib.reload(module) # Might be out of date during tests
     del sys.path[-1]
     return module

rhodecode/lib/dbmigrate/migrate/versioning/version.py

@@ -1,263 +1,263 b''
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 import os
 import re
 import shutil
 import logging
 
 from rhodecode.lib.dbmigrate.migrate import exceptions
 from rhodecode.lib.dbmigrate.migrate.versioning import pathed, script
 from datetime import datetime
 
 
 log = logging.getLogger(__name__)
 
 class VerNum(object):
     """A version number that behaves like a string and int at the same time"""
 
     _instances = {}
 
     def __new__(cls, value):
         val = str(value)
         if val not in cls._instances:
             cls._instances[val] = super(VerNum, cls).__new__(cls)
         ret = cls._instances[val]
         return ret
 
     def __init__(self,value):
         self.value = str(int(value))
         if self < 0:
             raise ValueError("Version number cannot be negative")
 
     def __add__(self, value):
         ret = int(self) + int(value)
         return VerNum(ret)
 
     def __sub__(self, value):
         return self + (int(value) * -1)
 
     def __eq__(self, value):
         return int(self) == int(value)
 
     def __ne__(self, value):
         return int(self) != int(value)
 
     def __lt__(self, value):
         return int(self) < int(value)
 
     def __gt__(self, value):
         return int(self) > int(value)
 
     def __ge__(self, value):
         return int(self) >= int(value)
 
     def __le__(self, value):
         return int(self) <= int(value)
 
     def __repr__(self):
         return "<VerNum(%s)>" % self.value
 
     def __str__(self):
         return str(self.value)
 
     def __int__(self):
         return int(self.value)
 
 
 class Collection(pathed.Pathed):
     """A collection of versioning scripts in a repository"""
 
     FILENAME_WITH_VERSION = re.compile(r'^(\d{3,}).*')
 
     def __init__(self, path):
         """Collect current version scripts in repository
         and store them in self.versions
         """
         super(Collection, self).__init__(path)
 
         # Create temporary list of files, allowing skipped version numbers.
         files = os.listdir(path)
         if '1' in files:
             # deprecation
             raise Exception('It looks like you have a repository in the old '
                 'format (with directories for each version). '
                 'Please convert repository before proceeding.')
 
         tempVersions = {}
         for filename in files:
             match = self.FILENAME_WITH_VERSION.match(filename)
             if match:
                 num = int(match.group(1))
                 tempVersions.setdefault(num, []).append(filename)
             else:
                 pass  # Must be a helper file or something, let's ignore it.
 
         # Create the versions member where the keys
         # are VerNum's and the values are Version's.
         self.versions = {}
-        for num, files in tempVersions.items():
+        for num, files in list(tempVersions.items()):
             self.versions[VerNum(num)] = Version(num, path, files)
 
     @property
     def latest(self):
         """:returns: Latest version in Collection"""
-        return max([VerNum(0)] + self.versions.keys())
+        return max([VerNum(0)] + list(self.versions.keys()))
 
     def _next_ver_num(self, use_timestamp_numbering):
         if use_timestamp_numbering == True:
             return VerNum(int(datetime.utcnow().strftime('%Y%m%d%H%M%S')))
         else:
             return self.latest + 1
 
     def create_new_python_version(self, description, **k):
         """Create Python files for new version"""
         ver = self._next_ver_num(k.pop('use_timestamp_numbering', False))
         extra = str_to_filename(description)
 
         if extra:
             if extra == '_':
                 extra = ''
             elif not extra.startswith('_'):
                 extra = '_%s' % extra
 
         filename = '%03d%s.py' % (ver, extra)
         filepath = self._version_path(filename)
 
         script.PythonScript.create(filepath, **k)
         self.versions[ver] = Version(ver, self.path, [filename])
 
     def create_new_sql_version(self, database, description, **k):
         """Create SQL files for new version"""
         ver = self._next_ver_num(k.pop('use_timestamp_numbering', False))
         self.versions[ver] = Version(ver, self.path, [])
 
         extra = str_to_filename(description)
 
         if extra:
             if extra == '_':
                 extra = ''
             elif not extra.startswith('_'):
                 extra = '_%s' % extra
 
         # Create new files.
         for op in ('upgrade', 'downgrade'):
             filename = '%03d%s_%s_%s.sql' % (ver, extra, database, op)
             filepath = self._version_path(filename)
             script.SqlScript.create(filepath, **k)
             self.versions[ver].add_script(filepath)
 
     def version(self, vernum=None):
         """Returns latest Version if vernum is not given.
         Otherwise, returns wanted version"""
         if vernum is None:
             vernum = self.latest
         return self.versions[VerNum(vernum)]
 
     @classmethod
     def clear(cls):
         super(Collection, cls).clear()
 
     def _version_path(self, ver):
         """Returns path of file in versions repository"""
         return os.path.join(self.path, str(ver))
 
 
 class Version(object):
     """A single version in a collection
     :param vernum: Version Number
     :param path: Path to script files
     :param filelist: List of scripts
     :type vernum: int, VerNum
     :type path: string
     :type filelist: list
     """
 
     def __init__(self, vernum, path, filelist):
         self.version = VerNum(vernum)
 
         # Collect scripts in this folder
         self.sql = {}
         self.python = None
 
         for script in filelist:
             self.add_script(os.path.join(path, script))
 
     def script(self, database=None, operation=None):
         """Returns SQL or Python Script"""
         for db in (database, 'default'):
             # Try to return a .sql script first
             try:
                 return self.sql[db][operation]
             except KeyError:
                 continue  # No .sql script exists
 
         # TODO: maybe add force Python parameter?
         ret = self.python
 
         assert ret is not None, \
             "There is no script for %d version" % self.version
         return ret
 
     def add_script(self, path):
         """Add script to Collection/Version"""
         if path.endswith(Extensions.py):
             self._add_script_py(path)
         elif path.endswith(Extensions.sql):
             self._add_script_sql(path)
 
     SQL_FILENAME = re.compile(r'^.*\.sql')
 
     def _add_script_sql(self, path):
         basename = os.path.basename(path)
         match = self.SQL_FILENAME.match(basename)
 
         if match:
             basename = basename.replace('.sql', '')
             parts = basename.split('_')
             if len(parts) < 3:
                 raise exceptions.ScriptError(
                     "Invalid SQL script name %s " % basename + \
                     "(needs to be ###_description_database_operation.sql)")
             version = parts[0]
             op = parts[-1]
             # NOTE(mriedem): check for ibm_db_sa as the database in the name
             if 'ibm_db_sa' in basename:
                 if len(parts) == 6:
                     dbms = '_'.join(parts[-4: -1])
                 else:
                     raise exceptions.ScriptError(
                         "Invalid ibm_db_sa SQL script name '%s'; "
                         "(needs to be "
                         "###_description_ibm_db_sa_operation.sql)" % basename)
             else:
                 dbms = parts[-2]
         else:
             raise exceptions.ScriptError(
                 "Invalid SQL script name %s " % basename + \
                 "(needs to be ###_description_database_operation.sql)")
 
         # File the script into a dictionary
         self.sql.setdefault(dbms, {})[op] = script.SqlScript(path)
 
     def _add_script_py(self, path):
         if self.python is not None:
             raise exceptions.ScriptError('You can only have one Python script '
                 'per version, but you have: %s and %s' % (self.python, path))
         self.python = script.PythonScript(path)
 
 
 class Extensions:
     """A namespace for file extensions"""
     py = 'py'
     sql = 'sql'
 
 def str_to_filename(s):
     """Replaces spaces, (double and single) quotes
     and double underscores to underscores
     """
 
     s = s.replace(' ', '_').replace('"', '_').replace("'", '_').replace(".", "_")
     while '__' in s:
         s = s.replace('__', '_')
     return s