dbmigrate: python3 fixes

rhodecode/lib/dbmigrate/migrate/versioning/cfgparse.py

@@ -1,27 +1,27 b''
 """
    Configuration parser module.
 """
 
-import configparser
+from configparser import ConfigParser
 
 from rhodecode.lib.dbmigrate.migrate.versioning.config import *
 from rhodecode.lib.dbmigrate.migrate.versioning import pathed
 
 
 class Parser(ConfigParser):
     """A project configuration file."""
 
     def to_dict(self, sections=None):
         """It's easier to access config values like dictionaries"""
         return self._sections
 
 
 class Config(pathed.Pathed, Parser):
     """Configuration class."""
 
     def __init__(self, path, *p, **k):
         """Confirm the config file exists; read it."""
         self.require_found(path)
         pathed.Pathed.__init__(self, path)
         Parser.__init__(self, *p, **k)
         self.read(path)

rhodecode/lib/dbmigrate/migrate/versioning/schemadiff.py

@@ -1,299 +1,299 b''
 """
    Schema differencing support.
 """
 
 import logging
 import sqlalchemy
 
 from sqlalchemy.types import Float
 
 log = logging.getLogger(__name__)
 
 
 def getDiffOfModelAgainstDatabase(metadata, engine, excludeTables=None):
     """
     Return differences of model against database.
 
     :return: object which will evaluate to :keyword:`True` if there \
       are differences else :keyword:`False`.
     """
     db_metadata = sqlalchemy.MetaData(engine)
     db_metadata.reflect()
 
     # sqlite will include a dynamically generated 'sqlite_sequence' table if
     # there are autoincrement sequences in the database; this should not be
     # compared.
     if engine.dialect.name == 'sqlite':
         if 'sqlite_sequence' in db_metadata.tables:
             db_metadata.remove(db_metadata.tables['sqlite_sequence'])
 
     return SchemaDiff(metadata, db_metadata,
                       labelA='model',
                       labelB='database',
                       excludeTables=excludeTables)
 
 
 def getDiffOfModelAgainstModel(metadataA, metadataB, excludeTables=None):
     """
     Return differences of model against another model.
 
     :return: object which will evaluate to :keyword:`True` if there \
       are differences else :keyword:`False`.
     """
     return SchemaDiff(metadataA, metadataB, excludeTables=excludeTables)
 
 
 class ColDiff(object):
     """
     Container for differences in one :class:`~sqlalchemy.schema.Column`
     between two :class:`~sqlalchemy.schema.Table` instances, ``A``
     and ``B``.
 
     .. attribute:: col_A
 
       The :class:`~sqlalchemy.schema.Column` object for A.
 
     .. attribute:: col_B
 
       The :class:`~sqlalchemy.schema.Column` object for B.
 
     .. attribute:: type_A
 
       The most generic type of the :class:`~sqlalchemy.schema.Column`
       object in A.
 
     .. attribute:: type_B
 
       The most generic type of the :class:`~sqlalchemy.schema.Column`
       object in A.
 
     """
 
     diff = False
 
     def __init__(self,col_A,col_B):
         self.col_A = col_A
         self.col_B = col_B
 
         self.type_A = col_A.type
         self.type_B = col_B.type
 
         self.affinity_A = self.type_A._type_affinity
         self.affinity_B = self.type_B._type_affinity
 
         if self.affinity_A is not self.affinity_B:
             self.diff = True
             return
 
         if isinstance(self.type_A,Float) or isinstance(self.type_B,Float):
             if not (isinstance(self.type_A,Float) and isinstance(self.type_B,Float)):
                 self.diff=True
                 return
 
         for attr in ('precision','scale','length'):
             A = getattr(self.type_A,attr,None)
             B = getattr(self.type_B,attr,None)
             if not (A is None or B is None) and A!=B:
                 self.diff=True
                 return
 
-    def __bool__(self):
+    def __nonzero__(self):
         return self.diff
 
     __bool__ = __nonzero__
 
 
 class TableDiff(object):
     """
     Container for differences in one :class:`~sqlalchemy.schema.Table`
     between two :class:`~sqlalchemy.schema.MetaData` instances, ``A``
     and ``B``.
 
     .. attribute:: columns_missing_from_A
 
       A sequence of column names that were found in B but weren't in
       A.
 
     .. attribute:: columns_missing_from_B
 
       A sequence of column names that were found in A but weren't in
       B.
 
     .. attribute:: columns_different
 
       A dictionary containing information about columns that were
       found to be different.
       It maps column names to a :class:`ColDiff` objects describing the
       differences found.
     """
     __slots__ = (
         'columns_missing_from_A',
         'columns_missing_from_B',
         'columns_different',
         )
 
-    def __bool__(self):
+    def __nonzero__(self):
         return bool(
             self.columns_missing_from_A or
             self.columns_missing_from_B or
             self.columns_different
             )
 
     __bool__ = __nonzero__
 
 class SchemaDiff(object):
     """
     Compute the difference between two :class:`~sqlalchemy.schema.MetaData`
     objects.
 
     The string representation of a :class:`SchemaDiff` will summarise
     the changes found between the two
     :class:`~sqlalchemy.schema.MetaData` objects.
 
     The length of a :class:`SchemaDiff` will give the number of
     changes found, enabling it to be used much like a boolean in
     expressions.
 
     :param metadataA:
       First :class:`~sqlalchemy.schema.MetaData` to compare.
 
     :param metadataB:
       Second :class:`~sqlalchemy.schema.MetaData` to compare.
 
     :param labelA:
       The label to use in messages about the first
       :class:`~sqlalchemy.schema.MetaData`.
 
     :param labelB:
       The label to use in messages about the second
       :class:`~sqlalchemy.schema.MetaData`.
 
     :param excludeTables:
       A sequence of table names to exclude.
 
     .. attribute:: tables_missing_from_A
 
       A sequence of table names that were found in B but weren't in
       A.
 
     .. attribute:: tables_missing_from_B
 
       A sequence of table names that were found in A but weren't in
       B.
 
     .. attribute:: tables_different
 
       A dictionary containing information about tables that were found
       to be different.
       It maps table names to a :class:`TableDiff` objects describing the
       differences found.
     """
 
     def __init__(self,
                  metadataA, metadataB,
                  labelA='metadataA',
                  labelB='metadataB',
                  excludeTables=None):
 
         self.metadataA, self.metadataB = metadataA, metadataB
         self.labelA, self.labelB = labelA, labelB
         self.label_width = max(len(labelA),len(labelB))
         excludeTables = set(excludeTables or [])
 
         A_table_names = set(metadataA.tables.keys())
         B_table_names = set(metadataB.tables.keys())
 
         self.tables_missing_from_A = sorted(
             B_table_names - A_table_names - excludeTables
             )
         self.tables_missing_from_B = sorted(
             A_table_names - B_table_names - excludeTables
             )
 
         self.tables_different = {}
         for table_name in A_table_names.intersection(B_table_names):
 
             td = TableDiff()
 
             A_table = metadataA.tables[table_name]
             B_table = metadataB.tables[table_name]
 
             A_column_names = set(A_table.columns.keys())
             B_column_names = set(B_table.columns.keys())
 
             td.columns_missing_from_A = sorted(
                 B_column_names - A_column_names
                 )
 
             td.columns_missing_from_B = sorted(
                 A_column_names - B_column_names
                 )
 
             td.columns_different = {}
 
             for col_name in A_column_names.intersection(B_column_names):
 
                 cd = ColDiff(
                     A_table.columns.get(col_name),
                     B_table.columns.get(col_name)
                     )
 
                 if cd:
                     td.columns_different[col_name]=cd
 
             # XXX - index and constraint differences should
             #       be checked for here
 
             if td:
                 self.tables_different[table_name]=td
 
     def __str__(self):
         """ Summarize differences. """
         out = []
         column_template ='      %%%is: %%r' % self.label_width
 
         for names,label in (
             (self.tables_missing_from_A,self.labelA),
             (self.tables_missing_from_B,self.labelB),
             ):
             if names:
                 out.append(
                     '  tables missing from %s: %s' % (
                         label,', '.join(sorted(names))
                         )
                     )
 
         for name,td in sorted(self.tables_different.items()):
             out.append(
                '  table with differences: %s' % name
                )
             for names,label in (
                 (td.columns_missing_from_A,self.labelA),
                 (td.columns_missing_from_B,self.labelB),
                 ):
                 if names:
                     out.append(
                         '    %s missing these columns: %s' % (
                             label,', '.join(sorted(names))
                             )
                         )
             for name,cd in list(td.columns_different.items()):
                 out.append('    column with differences: %s' % name)
                 out.append(column_template % (self.labelA,cd.col_A))
                 out.append(column_template % (self.labelB,cd.col_B))
 
         if out:
             out.insert(0, 'Schema diffs:')
             return '\n'.join(out)
         else:
             return 'No schema diffs'
 
     def __len__(self):
         """
         Used in bool evaluation, return of 0 means no diffs.
         """
         return (
             len(self.tables_missing_from_A) +
             len(self.tables_missing_from_B) +
             len(self.tables_different)
             )

rhodecode/lib/dbmigrate/migrate/versioning/version.py

@@ -1,263 +1,269 b''
 #!/usr/bin/env python
 
 
 import os
 import re
 import shutil
 import logging
 
 from rhodecode.lib.dbmigrate.migrate import exceptions
 from rhodecode.lib.dbmigrate.migrate.versioning import pathed, script
 from datetime import datetime
 
 
 log = logging.getLogger(__name__)
 
 class VerNum(object):
     """A version number that behaves like a string and int at the same time"""
 
     _instances = {}
 
     def __new__(cls, value):
         val = str(value)
         if val not in cls._instances:
             cls._instances[val] = super(VerNum, cls).__new__(cls)
         ret = cls._instances[val]
         return ret
 
     def __init__(self,value):
         self.value = str(int(value))
         if self < 0:
             raise ValueError("Version number cannot be negative")
 
     def __add__(self, value):
         ret = int(self) + int(value)
         return VerNum(ret)
 
     def __sub__(self, value):
         return self + (int(value) * -1)
 
     def __eq__(self, value):
         return int(self) == int(value)
 
     def __ne__(self, value):
         return int(self) != int(value)
 
     def __lt__(self, value):
         return int(self) < int(value)
 
     def __gt__(self, value):
         return int(self) > int(value)
 
     def __ge__(self, value):
         return int(self) >= int(value)
 
     def __le__(self, value):
         return int(self) <= int(value)
 
     def __repr__(self):
         return "<VerNum(%s)>" % self.value
 
     def __str__(self):
         return str(self.value)
 
     def __int__(self):
         return int(self.value)
 
+    def __index__(self):
+        return int(self.value)
+
+    def __hash__(self):
+        return hash(self.value)
+
 
 class Collection(pathed.Pathed):
     """A collection of versioning scripts in a repository"""
 
     FILENAME_WITH_VERSION = re.compile(r'^(\d{3,}).*')
 
     def __init__(self, path):
         """Collect current version scripts in repository
         and store them in self.versions
         """
         super(Collection, self).__init__(path)
 
         # Create temporary list of files, allowing skipped version numbers.
         files = os.listdir(path)
         if '1' in files:
             # deprecation
             raise Exception('It looks like you have a repository in the old '
                 'format (with directories for each version). '
                 'Please convert repository before proceeding.')
 
         tempVersions = {}
         for filename in files:
             match = self.FILENAME_WITH_VERSION.match(filename)
             if match:
                 num = int(match.group(1))
                 tempVersions.setdefault(num, []).append(filename)
             else:
                 pass  # Must be a helper file or something, let's ignore it.
 
         # Create the versions member where the keys
         # are VerNum's and the values are Version's.
         self.versions = {}
         for num, files in list(tempVersions.items()):
             self.versions[VerNum(num)] = Version(num, path, files)
 
     @property
     def latest(self):
         """:returns: Latest version in Collection"""
         return max([VerNum(0)] + list(self.versions.keys()))
 
     def _next_ver_num(self, use_timestamp_numbering):
         if use_timestamp_numbering is True:
             return VerNum(int(datetime.utcnow().strftime('%Y%m%d%H%M%S')))
         else:
             return self.latest + 1
 
     def create_new_python_version(self, description, **k):
         """Create Python files for new version"""
         ver = self._next_ver_num(k.pop('use_timestamp_numbering', False))
         extra = str_to_filename(description)
 
         if extra:
             if extra == '_':
                 extra = ''
             elif not extra.startswith('_'):
                 extra = '_%s' % extra
 
         filename = '%03d%s.py' % (ver, extra)
         filepath = self._version_path(filename)
 
         script.PythonScript.create(filepath, **k)
         self.versions[ver] = Version(ver, self.path, [filename])
 
     def create_new_sql_version(self, database, description, **k):
         """Create SQL files for new version"""
         ver = self._next_ver_num(k.pop('use_timestamp_numbering', False))
         self.versions[ver] = Version(ver, self.path, [])
 
         extra = str_to_filename(description)
 
         if extra:
             if extra == '_':
                 extra = ''
             elif not extra.startswith('_'):
                 extra = '_%s' % extra
 
         # Create new files.
         for op in ('upgrade', 'downgrade'):
             filename = '%03d%s_%s_%s.sql' % (ver, extra, database, op)
             filepath = self._version_path(filename)
             script.SqlScript.create(filepath, **k)
             self.versions[ver].add_script(filepath)
 
     def version(self, vernum=None):
         """Returns latest Version if vernum is not given.
         Otherwise, returns wanted version"""
         if vernum is None:
             vernum = self.latest
         return self.versions[VerNum(vernum)]
 
     @classmethod
     def clear(cls):
         super(Collection, cls).clear()
 
     def _version_path(self, ver):
         """Returns path of file in versions repository"""
         return os.path.join(self.path, str(ver))
 
 
 class Version(object):
     """A single version in a collection
     :param vernum: Version Number
     :param path: Path to script files
     :param filelist: List of scripts
     :type vernum: int, VerNum
     :type path: string
     :type filelist: list
     """
 
     def __init__(self, vernum, path, filelist):
         self.version = VerNum(vernum)
 
         # Collect scripts in this folder
         self.sql = {}
         self.python = None
 
         for script in filelist:
             self.add_script(os.path.join(path, script))
 
     def script(self, database=None, operation=None):
         """Returns SQL or Python Script"""
         for db in (database, 'default'):
             # Try to return a .sql script first
             try:
                 return self.sql[db][operation]
             except KeyError:
                 continue  # No .sql script exists
 
         # TODO: maybe add force Python parameter?
         ret = self.python
 
         assert ret is not None, \
             "There is no script for %d version" % self.version
         return ret
 
     def add_script(self, path):
         """Add script to Collection/Version"""
         if path.endswith(Extensions.py):
             self._add_script_py(path)
         elif path.endswith(Extensions.sql):
             self._add_script_sql(path)
 
     SQL_FILENAME = re.compile(r'^.*\.sql')
 
     def _add_script_sql(self, path):
         basename = os.path.basename(path)
         match = self.SQL_FILENAME.match(basename)
 
         if match:
             basename = basename.replace('.sql', '')
             parts = basename.split('_')
             if len(parts) < 3:
                 raise exceptions.ScriptError(
                     "Invalid SQL script name %s " % basename + \
                     "(needs to be ###_description_database_operation.sql)")
             version = parts[0]
             op = parts[-1]
             # NOTE(mriedem): check for ibm_db_sa as the database in the name
             if 'ibm_db_sa' in basename:
                 if len(parts) == 6:
                     dbms = '_'.join(parts[-4: -1])
                 else:
                     raise exceptions.ScriptError(
                         "Invalid ibm_db_sa SQL script name '%s'; "
                         "(needs to be "
                         "###_description_ibm_db_sa_operation.sql)" % basename)
             else:
                 dbms = parts[-2]
         else:
             raise exceptions.ScriptError(
                 "Invalid SQL script name %s " % basename + \
                 "(needs to be ###_description_database_operation.sql)")
 
         # File the script into a dictionary
         self.sql.setdefault(dbms, {})[op] = script.SqlScript(path)
 
     def _add_script_py(self, path):
         if self.python is not None:
             raise exceptions.ScriptError('You can only have one Python script '
                 'per version, but you have: %s and %s' % (self.python, path))
         self.python = script.PythonScript(path)
 
 
 class Extensions:
     """A namespace for file extensions"""
     py = 'py'
     sql = 'sql'
 
 def str_to_filename(s):
     """Replaces spaces, (double and single) quotes
     and double underscores to underscores
     """
 
     s = s.replace(' ', '_').replace('"', '_').replace("'", '_').replace(".", "_")
     while '__' in s:
         s = s.replace('__', '_')
     return s