update migrations for 1.2

rhodecode/lib/dbmigrate/migrate/__init__.py

@@ -7,3 +7,5 b''
 
 from rhodecode.lib.dbmigrate.migrate.versioning import *
 from rhodecode.lib.dbmigrate.migrate.changeset import *
+
+__version__ = '0.7.2.dev'
  No newline at end of file

rhodecode/lib/dbmigrate/migrate/changeset/__init__.py

@@ -12,9 +12,10 b' from sqlalchemy import __version__ as _s'
 
 warnings.simplefilter('always', DeprecationWarning)
 
-_sa_version = tuple(int(re.match("\d+", x).group(0))
+_sa_version = tuple(int(re.match("\d+", x).group(0)) 
                     for x in _sa_version.split("."))
 SQLA_06 = _sa_version >= (0, 6)
+SQLA_07 = _sa_version >= (0, 7)
 
 del re
 del _sa_version

rhodecode/lib/dbmigrate/migrate/changeset/schema.py

@@ -11,9 +11,9 b' from sqlalchemy.schema import ForeignKey'
 from sqlalchemy.schema import UniqueConstraint
 
 from rhodecode.lib.dbmigrate.migrate.exceptions import *
-from rhodecode.lib.dbmigrate.migrate.changeset import SQLA_06
+from rhodecode.lib.dbmigrate.migrate.changeset import SQLA_06, SQLA_07
 from rhodecode.lib.dbmigrate.migrate.changeset.databases.visitor import (get_engine_visitor,
-                                                 run_single_visitor)
+                                                                         run_single_visitor)
 
 
 __all__ = [
@@ -555,7 +555,10 b' populated with defaults'
 
     def add_to_table(self, table):
         if table is not None  and self.table is None:
-            self._set_parent(table)
+            if SQLA_07:
+                table.append_column(self)
+            else:
+                self._set_parent(table)
 
     def _col_name_in_constraint(self,cons,name):
         return False
@@ -590,7 +593,10 b' populated with defaults'
         table.constraints = table.constraints - to_drop
 
         if table.c.contains_column(self):
-            table.c.remove(self)
+            if SQLA_07:
+                table._columns.remove(self)
+            else:
+                table.c.remove(self)
 
     # TODO: this is fixed in 0.6
     def copy_fixed(self, **kw):

rhodecode/lib/dbmigrate/migrate/exceptions.py

@@ -71,6 +71,11 b' class InvalidScriptError(ScriptError):'
     """Invalid script error."""
 
 
+class InvalidVersionError(Error):
+    """Invalid version error."""
+
+# migrate.changeset
+
 class NotSupportedError(Error):
     """Not supported error"""
 

rhodecode/lib/dbmigrate/migrate/versioning/api.py

@@ -110,19 +110,19 b' def script(description, repository, **op'
 
 
 @catch_known_errors
-def script_sql(database, repository, **opts):
-    """%prog script_sql DATABASE REPOSITORY_PATH
+def script_sql(database, description, repository, **opts):
+    """%prog script_sql DATABASE DESCRIPTION REPOSITORY_PATH
 
     Create empty change SQL scripts for given DATABASE, where DATABASE
-    is either specific ('postgres', 'mysql', 'oracle', 'sqlite', etc.)
+    is either specific ('postgresql', 'mysql', 'oracle', 'sqlite', etc.)
     or generic ('default').
 
-    For instance, manage.py script_sql postgres creates:
-    repository/versions/001_postgres_upgrade.sql and
-    repository/versions/001_postgres_postgres.sql
+    For instance, manage.py script_sql postgresql description creates:
+    repository/versions/001_description_postgresql_upgrade.sql and
+    repository/versions/001_description_postgresql_postgres.sql
     """
     repo = Repository(repository)
-    repo.create_script_sql(database, **opts)
+    repo.create_script_sql(database, description, **opts)
 
 
 def version(repository, **opts):

rhodecode/lib/dbmigrate/migrate/versioning/genmodel.py

@@ -1,9 +1,9 b''
 """
-   Code to generate a Python model from a database or differences
-   between a model and database.
+Code to generate a Python model from a database or differences
+between a model and database.
 
-   Some of this is borrowed heavily from the AutoCode project at:
-   http://code.google.com/p/sqlautocode/
+Some of this is borrowed heavily from the AutoCode project at:
+http://code.google.com/p/sqlautocode/
 """
 
 import sys
@@ -14,6 +14,7 b' import sqlalchemy'
 from rhodecode.lib.dbmigrate import migrate
 from rhodecode.lib.dbmigrate.migrate import changeset
 
+
 log = logging.getLogger(__name__)
 HEADER = """
 ## File autogenerated by genmodel.py
@@ -33,6 +34,13 b' Base = declarative.declarative_base()'
 
 
 class ModelGenerator(object):
+    """Various transformations from an A, B diff.
+
+    In the implementation, A tends to be called the model and B
+    the database (although this is not true of all diffs).
+    The diff is directionless, but transformations apply the diff
+    in a particular direction, described in the method name.
+    """
 
     def __init__(self, diff, engine, declarative=False):
         self.diff = diff
@@ -58,7 +66,7 b' class ModelGenerator(object):'
                 pass
             else:
                 kwarg.append('default')
-        ks = ', '.join('%s=%r' % (k, getattr(col, k)) for k in kwarg)
+        args = ['%s=%r' % (k, getattr(col, k)) for k in kwarg]
 
         # crs: not sure if this is good idea, but it gets rid of extra
         # u''
@@ -72,43 +80,38 b' class ModelGenerator(object):'
                     type_ = cls()
                 break
 
+        type_repr = repr(type_)
+        if type_repr.endswith('()'):
+            type_repr = type_repr[:-2]
+
+        constraints = [repr(cn) for cn in col.constraints]
+
         data = {
             'name': name,
-            'type': type_,
-            'constraints': ', '.join([repr(cn) for cn in col.constraints]),
-            'args': ks and ks or ''}
+            'commonStuff': ', '.join([type_repr] + constraints + args),
+        }
 
-        if data['constraints']:
-            if data['args']:
-                data['args'] = ',' + data['args']
-
-        if data['constraints'] or data['args']:
-            data['maybeComma'] = ','
+        if self.declarative:
+            return """%(name)s = Column(%(commonStuff)s)""" % data
         else:
-            data['maybeComma'] = ''
+            return """Column(%(name)r, %(commonStuff)s)""" % data
 
-        commonStuff = """ %(maybeComma)s %(constraints)s %(args)s)""" % data
-        commonStuff = commonStuff.strip()
-        data['commonStuff'] = commonStuff
-        if self.declarative:
-            return """%(name)s = Column(%(type)r%(commonStuff)s""" % data
-        else:
-            return """Column(%(name)r, %(type)r%(commonStuff)s""" % data
-
-    def getTableDefn(self, table):
+    def _getTableDefn(self, table, metaName='meta'):
         out = []
         tableName = table.name
         if self.declarative:
             out.append("class %(table)s(Base):" % {'table': tableName})
-            out.append("  __tablename__ = '%(table)s'" % {'table': tableName})
+            out.append("    __tablename__ = '%(table)s'\n" %
+                            {'table': tableName})
             for col in table.columns:
-                out.append("  %s" % self.column_repr(col))
+                out.append("    %s" % self.column_repr(col))
+            out.append('\n')
         else:
-            out.append("%(table)s = Table('%(table)s', meta," % \
-                           {'table': tableName})
+            out.append("%(table)s = Table('%(table)s', %(meta)s," %
+                       {'table': tableName, 'meta': metaName})
             for col in table.columns:
-                out.append("  %s," % self.column_repr(col))
-            out.append(")")
+                out.append("    %s," % self.column_repr(col))
+            out.append(")\n")
         return out
 
     def _get_tables(self,missingA=False,missingB=False,modified=False):
@@ -122,8 +125,14 b' class ModelGenerator(object):'
                 for name in names:
                     yield metadata.tables.get(name)
 
-    def toPython(self):
-        """Assume database is current and model is empty."""
+    def genBDefinition(self):
+        """Generates the source code for a definition of B.
+
+        Assumes a diff where A is empty.
+
+        Was: toPython. Assume database (B) is current and model (A) is empty.
+        """
+
         out = []
         if self.declarative:
             out.append(DECLARATIVE_HEADER)
@@ -131,67 +140,89 b' class ModelGenerator(object):'
             out.append(HEADER)
         out.append("")
         for table in self._get_tables(missingA=True):
-            out.extend(self.getTableDefn(table))
-            out.append("")
+            out.extend(self._getTableDefn(table))
         return '\n'.join(out)
 
-    def toUpgradeDowngradePython(self, indent='    '):
-        ''' Assume model is most current and database is out-of-date. '''
-        decls = ['from rhodecode.lib.dbmigrate.migrate.changeset import schema',
-                 'meta = MetaData()']
-        for table in self._get_tables(
-            missingA=True,missingB=True,modified=True
-            ):
-            decls.extend(self.getTableDefn(table))
+    def genB2AMigration(self, indent='    '):
+        '''Generate a migration from B to A.
+
+        Was: toUpgradeDowngradePython
+        Assume model (A) is most current and database (B) is out-of-date.
+        '''
+
+        decls = ['from migrate.changeset import schema',
+                 'pre_meta = MetaData()',
+                 'post_meta = MetaData()',
+                ]
+        upgradeCommands = ['pre_meta.bind = migrate_engine',
+                           'post_meta.bind = migrate_engine']
+        downgradeCommands = list(upgradeCommands)
+
+        for tn in self.diff.tables_missing_from_A:
+            pre_table = self.diff.metadataB.tables[tn]
+            decls.extend(self._getTableDefn(pre_table, metaName='pre_meta'))
+            upgradeCommands.append(
+                "pre_meta.tables[%(table)r].drop()" % {'table': tn})
+            downgradeCommands.append(
+                "pre_meta.tables[%(table)r].create()" % {'table': tn})
 
-        upgradeCommands, downgradeCommands = [], []
-        for tableName in self.diff.tables_missing_from_A:
-            upgradeCommands.append("%(table)s.drop()" % {'table': tableName})
-            downgradeCommands.append("%(table)s.create()" % \
-                                         {'table': tableName})
-        for tableName in self.diff.tables_missing_from_B:
-            upgradeCommands.append("%(table)s.create()" % {'table': tableName})
-            downgradeCommands.append("%(table)s.drop()" % {'table': tableName})
+        for tn in self.diff.tables_missing_from_B:
+            post_table = self.diff.metadataA.tables[tn]
+            decls.extend(self._getTableDefn(post_table, metaName='post_meta'))
+            upgradeCommands.append(
+                "post_meta.tables[%(table)r].create()" % {'table': tn})
+            downgradeCommands.append(
+                "post_meta.tables[%(table)r].drop()" % {'table': tn})
 
-        for tableName in self.diff.tables_different:
-            dbTable = self.diff.metadataB.tables[tableName]
-            missingInDatabase, missingInModel, diffDecl = \
-                self.diff.colDiffs[tableName]
-            for col in missingInDatabase:
-                upgradeCommands.append('%s.columns[%r].create()' % (
-                        modelTable, col.name))
-                downgradeCommands.append('%s.columns[%r].drop()' % (
-                        modelTable, col.name))
-            for col in missingInModel:
-                upgradeCommands.append('%s.columns[%r].drop()' % (
-                        modelTable, col.name))
-                downgradeCommands.append('%s.columns[%r].create()' % (
-                        modelTable, col.name))
-            for modelCol, databaseCol, modelDecl, databaseDecl in diffDecl:
+        for (tn, td) in self.diff.tables_different.iteritems():
+            if td.columns_missing_from_A or td.columns_different:
+                pre_table = self.diff.metadataB.tables[tn]
+                decls.extend(self._getTableDefn(
+                    pre_table, metaName='pre_meta'))
+            if td.columns_missing_from_B or td.columns_different:
+                post_table = self.diff.metadataA.tables[tn]
+                decls.extend(self._getTableDefn(
+                    post_table, metaName='post_meta'))
+
+            for col in td.columns_missing_from_A:
+                upgradeCommands.append(
+                    'pre_meta.tables[%r].columns[%r].drop()' % (tn, col))
+                downgradeCommands.append(
+                    'pre_meta.tables[%r].columns[%r].create()' % (tn, col))
+            for col in td.columns_missing_from_B:
+                upgradeCommands.append(
+                    'post_meta.tables[%r].columns[%r].create()' % (tn, col))
+                downgradeCommands.append(
+                    'post_meta.tables[%r].columns[%r].drop()' % (tn, col))
+            for modelCol, databaseCol, modelDecl, databaseDecl in td.columns_different:
                 upgradeCommands.append(
                     'assert False, "Can\'t alter columns: %s:%s=>%s"' % (
-                    modelTable, modelCol.name, databaseCol.name))
+                    tn, modelCol.name, databaseCol.name))
                 downgradeCommands.append(
                     'assert False, "Can\'t alter columns: %s:%s=>%s"' % (
-                    modelTable, modelCol.name, databaseCol.name))
-        pre_command = '    meta.bind = migrate_engine'
+                    tn, modelCol.name, databaseCol.name))
 
         return (
             '\n'.join(decls),
-            '\n'.join([pre_command] + ['%s%s' % (indent, line) for line in upgradeCommands]),
-            '\n'.join([pre_command] + ['%s%s' % (indent, line) for line in downgradeCommands]))
+            '\n'.join('%s%s' % (indent, line) for line in upgradeCommands),
+            '\n'.join('%s%s' % (indent, line) for line in downgradeCommands))
 
     def _db_can_handle_this_change(self,td):
+        """Check if the database can handle going from B to A."""
+
         if (td.columns_missing_from_B
             and not td.columns_missing_from_A
             and not td.columns_different):
-            # Even sqlite can handle this.
+            # Even sqlite can handle column additions.
             return True
         else:
             return not self.engine.url.drivername.startswith('sqlite')
 
-    def applyModel(self):
-        """Apply model to current database."""
+    def runB2A(self):
+        """Goes from B to A.
+
+        Was: applyModel. Apply model (A) to current database (B).
+        """
 
         meta = sqlalchemy.MetaData(self.engine)
 
@@ -251,3 +282,4 b' class ModelGenerator(object):'
                 except:
                     trans.rollback()
                     raise
+

rhodecode/lib/dbmigrate/migrate/versioning/repository.py

@@ -115,6 +115,7 b' class Repository(pathed.Pathed):'
         options.setdefault('version_table', 'migrate_version')
         options.setdefault('repository_id', name)
         options.setdefault('required_dbs', [])
+        options.setdefault('use_timestamp_numbering', '0')
 
         tmpl = open(os.path.join(tmpl_dir, cls._config)).read()
         ret = TempitaTemplate(tmpl).substitute(options)
@@ -152,11 +153,14 b' class Repository(pathed.Pathed):'
 
     def create_script(self, description, **k):
         """API to :meth:`migrate.versioning.version.Collection.create_new_python_version`"""
+        
+        k['use_timestamp_numbering'] = self.use_timestamp_numbering
         self.versions.create_new_python_version(description, **k)
 
-    def create_script_sql(self, database, **k):
+    def create_script_sql(self, database, description, **k):
         """API to :meth:`migrate.versioning.version.Collection.create_new_sql_version`"""
-        self.versions.create_new_sql_version(database, **k)
+        k['use_timestamp_numbering'] = self.use_timestamp_numbering
+        self.versions.create_new_sql_version(database, description, **k)
 
     @property
     def latest(self):
@@ -173,6 +177,13 b' class Repository(pathed.Pathed):'
         """Returns repository id specified in config"""
         return self.config.get('db_settings', 'repository_id')
 
+    @property
+    def use_timestamp_numbering(self):
+        """Returns use_timestamp_numbering specified in config"""
+        ts_numbering = self.config.get('db_settings', 'use_timestamp_numbering', raw=True)
+        
+        return ts_numbering
+
     def version(self, *p, **k):
         """API to :attr:`migrate.versioning.version.Collection.version`"""
         return self.versions.version(*p, **k)

rhodecode/lib/dbmigrate/migrate/versioning/schema.py

@@ -11,6 +11,7 b' from sqlalchemy import exceptions as sa_'
 from sqlalchemy.sql import bindparam
 
 from rhodecode.lib.dbmigrate.migrate import exceptions
+from rhodecode.lib.dbmigrate.migrate.changeset import SQLA_07
 from rhodecode.lib.dbmigrate.migrate.versioning import genmodel, schemadiff
 from rhodecode.lib.dbmigrate.migrate.versioning.repository import Repository
 from rhodecode.lib.dbmigrate.migrate.versioning.util import load_model
@@ -57,10 +58,16 b' class ControlledSchema(object):'
         """
         Remove version control from a database.
         """
-        try:
-            self.table.drop()
-        except (sa_exceptions.SQLError):
-            raise exceptions.DatabaseNotControlledError(str(self.table))
+        if SQLA_07:
+            try:
+                self.table.drop()
+            except sa_exceptions.DatabaseError:
+                raise exceptions.DatabaseNotControlledError(str(self.table))
+        else:
+            try:
+                self.table.drop()
+            except (sa_exceptions.SQLError):
+                raise exceptions.DatabaseNotControlledError(str(self.table))
 
     def changeset(self, version=None):
         """API to Changeset creation.
@@ -110,7 +117,7 b' class ControlledSchema(object):'
         diff = schemadiff.getDiffOfModelAgainstDatabase(
             model, self.engine, excludeTables=[self.repository.version_table]
             )
-        genmodel.ModelGenerator(diff,self.engine).applyModel()
+        genmodel.ModelGenerator(diff,self.engine).runB2A()
 
         self.update_repository_table(self.version, int(self.repository.latest))
 
@@ -210,4 +217,4 b' class ControlledSchema(object):'
         diff = schemadiff.getDiffOfModelAgainstDatabase(
             MetaData(), engine, excludeTables=[repository.version_table]
             )
-        return genmodel.ModelGenerator(diff, engine, declarative).toPython()
+        return genmodel.ModelGenerator(diff, engine, declarative).genBDefinition()

rhodecode/lib/dbmigrate/migrate/versioning/script/py.py

@@ -61,12 +61,12 b' class PythonScript(base.BaseScript):'
 
         # Compute differences.
         diff = schemadiff.getDiffOfModelAgainstModel(
+            model,
             oldmodel,
-            model,
             excludeTables=[repository.version_table])
         # TODO: diff can be False (there is no difference?)
         decls, upgradeCommands, downgradeCommands = \
-            genmodel.ModelGenerator(diff,engine).toUpgradeDowngradePython()
+            genmodel.ModelGenerator(diff,engine).genB2AMigration()
 
         # Store differences into file.
         src = Template(opts.pop('templates_path', None)).get_script(opts.pop('templates_theme', None))

rhodecode/lib/dbmigrate/migrate/versioning/templates/repository/default/migrate.cfg

@@ -18,3 +18,8 b" version_table={{ locals().pop('version_t"
 # be using to ensure your updates to that database work properly.
 # This must be a list; example: ['postgres','sqlite']
 required_dbs={{ locals().pop('required_dbs') }}
+
+# When creating new change scripts, Migrate will stamp the new script with
+# a version number. By default this is latest_version + 1. You can set this
+# to 'true' to tell Migrate to use the UTC timestamp instead.
+use_timestamp_numbering='false'
  No newline at end of file

rhodecode/lib/dbmigrate/migrate/versioning/version.py

@@ -8,6 +8,7 b' import logging'
 
 from rhodecode.lib.dbmigrate.migrate import exceptions
 from rhodecode.lib.dbmigrate.migrate.versioning import pathed, script
+from datetime import datetime
 
 
 log = logging.getLogger(__name__)
@@ -59,7 +60,7 b' class Collection(pathed.Pathed):'
         and store them in self.versions
         """
         super(Collection, self).__init__(path)
-
+        
         # Create temporary list of files, allowing skipped version numbers.
         files = os.listdir(path)
         if '1' in files:
@@ -88,9 +89,17 b' class Collection(pathed.Pathed):'
         """:returns: Latest version in Collection"""
         return max([VerNum(0)] + self.versions.keys())
 
+    def _next_ver_num(self, use_timestamp_numbering):
+        print use_timestamp_numbering
+        if use_timestamp_numbering == True:
+            print "Creating new timestamp version!"
+            return VerNum(int(datetime.utcnow().strftime('%Y%m%d%H%M%S')))
+        else:
+            return self.latest + 1
+
     def create_new_python_version(self, description, **k):
         """Create Python files for new version"""
-        ver = self.latest + 1
+        ver = self._next_ver_num(k.pop('use_timestamp_numbering', False))
         extra = str_to_filename(description)
 
         if extra:
@@ -104,19 +113,27 b' class Collection(pathed.Pathed):'
 
         script.PythonScript.create(filepath, **k)
         self.versions[ver] = Version(ver, self.path, [filename])
-
-    def create_new_sql_version(self, database, **k):
+        
+    def create_new_sql_version(self, database, description, **k):
         """Create SQL files for new version"""
-        ver = self.latest + 1
+        ver = self._next_ver_num(k.pop('use_timestamp_numbering', False))
         self.versions[ver] = Version(ver, self.path, [])
 
+        extra = str_to_filename(description)
+
+        if extra:
+            if extra == '_':
+                extra = ''
+            elif not extra.startswith('_'):
+                extra = '_%s' % extra
+
         # Create new files.
         for op in ('upgrade', 'downgrade'):
-            filename = '%03d_%s_%s.sql' % (ver, database, op)
+            filename = '%03d%s_%s_%s.sql' % (ver, extra, database, op)
             filepath = self._version_path(filename)
             script.SqlScript.create(filepath, **k)
             self.versions[ver].add_script(filepath)
-
+        
     def version(self, vernum=None):
         """Returns latest Version if vernum is not given.
         Otherwise, returns wanted version"""
@@ -135,7 +152,7 b' class Collection(pathed.Pathed):'
 
 class Version(object):
     """A single version in a collection
-    :param vernum: Version Number
+    :param vernum: Version Number 
     :param path: Path to script files
     :param filelist: List of scripts
     :type vernum: int, VerNum
@@ -152,7 +169,7 b' class Version(object):'
 
         for script in filelist:
             self.add_script(os.path.join(path, script))
-
+    
     def script(self, database=None, operation=None):
         """Returns SQL or Python Script"""
         for db in (database, 'default'):
@@ -176,18 +193,26 b' class Version(object):'
         elif path.endswith(Extensions.sql):
             self._add_script_sql(path)
 
-    SQL_FILENAME = re.compile(r'^(\d+)_([^_]+)_([^_]+).sql')
+    SQL_FILENAME = re.compile(r'^.*\.sql')
 
     def _add_script_sql(self, path):
         basename = os.path.basename(path)
         match = self.SQL_FILENAME.match(basename)
-
+        
         if match:
-            version, dbms, op = match.group(1), match.group(2), match.group(3)
+            basename = basename.replace('.sql', '')
+            parts = basename.split('_')
+            if len(parts) < 3:
+                raise exceptions.ScriptError(
+                    "Invalid SQL script name %s " % basename + \
+                    "(needs to be ###_description_database_operation.sql)")
+            version = parts[0]
+            op = parts[-1]
+            dbms = parts[-2]
         else:
             raise exceptions.ScriptError(
                 "Invalid SQL script name %s " % basename + \
-                "(needs to be ###_database_operation.sql)")
+                "(needs to be ###_description_database_operation.sql)")
 
         # File the script into a dictionary
         self.sql.setdefault(dbms, {})[op] = script.SqlScript(path)

rhodecode/lib/dbmigrate/versions/003_version_1_2_0.py

@@ -80,6 +80,11 b' def upgrade(migrate_engine):'
     enable_downloads = Column("downloads", Boolean(), nullable=True, unique=None, default=True)
     enable_downloads.create(Repository().__table__)
 
+    #ADD column created_on
+    created_on = Column('created_on', DateTime(timezone=False), nullable=True,
+                        unique=None, default=datetime.datetime.now)
+    created_on.create(Repository().__table__)
+
     #ADD group_id column#
     group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'),
                   nullable=True, unique=False, default=None)
@@ -94,6 +99,15 b' def upgrade(migrate_engine):'
                         nullable=True, unique=False, default=None)
 
     clone_uri.create(Repository().__table__)
+
+
+    #==========================================================================
+    # Upgrade of `user_followings` table
+    #==========================================================================
+
+    follows_from = Column('follows_from', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)
+    follows_from.create(Repository().__table__)
+
     return