dbmigrations:

added first working upgrade script
fixed wrong versions
fixed template path

rhodecode/__init__.py

@@ -1,48 +1,48
 # -*- coding: utf-8 -*-
 """
     rhodecode.__init__
     ~~~~~~~~~~~~~~~~~~
 
     RhodeCode, a web based repository management based on pylons
     versioning implementation: http://semver.org/
 
     :created_on: Apr 9, 2010
     :author: marcink
     :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
 
 VERSION = (1, 1, 0, 'beta')
 __version__ = '.'.join((str(each) for each in VERSION[:4]))
-__dbversion__ = 1 #defines current db version for migrations
+__dbversion__ = 2 #defines current db version for migrations
 
 from rhodecode.lib.utils import get_current_revision
 _rev = get_current_revision()
 
 if len(VERSION) > 3 and _rev:
     __version__ += ' [rev:%s]' % _rev[0]
 
 def get_version():
     """Returns shorter version (digit parts only) as string."""
 
     return '.'.join((str(each) for each in VERSION[:3]))
 
 BACKENDS = {
     'hg': 'Mercurial repository',
    #'git': 'Git repository',
 }

rhodecode/lib/dbmigrate/__init__.py

@@ -1,77 +1,86
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.dbmigrate.__init__
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     
     Database migration modules
     
     :created_on: Dec 11, 2010
     :author: marcink
     :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
 import logging
 from sqlalchemy import engine_from_config
 
+from rhodecode import __dbversion__
+from rhodecode.lib.dbmigrate.migrate.versioning import api
 from rhodecode.lib.dbmigrate.migrate.exceptions import \
     DatabaseNotControlledError
 from rhodecode.lib.utils import BasePasterCommand, Command, add_cache
 
 log = logging.getLogger(__name__)
 
 class UpgradeDb(BasePasterCommand):
     """Command used for paster to upgrade our database to newer version
     """
 
     max_args = 1
     min_args = 1
 
     usage = "CONFIG_FILE"
     summary = "Upgrades current db to newer version given configuration file"
     group_name = "RhodeCode"
 
     parser = Command.standard_parser(verbose=True)
 
     def command(self):
         from pylons import config
+
         add_cache(config)
-        #engine = engine_from_config(config, 'sqlalchemy.db1.')
-        #rint engine
 
-        from rhodecode.lib.dbmigrate.migrate.versioning import api
-        path = 'rhodecode/lib/dbmigrate'
+        #engine = engine_from_config(config, 'sqlalchemy.db1.')
 
+        repository_path = 'rhodecode/lib/dbmigrate'
+        db_uri = config['sqlalchemy.db1.url']
 
         try:
-            curr_version = api.db_version(config['sqlalchemy.db1.url'], path)
+            curr_version = api.db_version(db_uri, repository_path)
             msg = ('Found current database under version'
                  ' control with version %s' % curr_version)
 
         except (RuntimeError, DatabaseNotControlledError), e:
-            curr_version = 0
+            curr_version = 1
             msg = ('Current database is not under version control setting'
                    ' as version %s' % curr_version)
+            api.version_control(db_uri, repository_path, curr_version)
+
 
         print msg
+        #now we have our dbversion we can do upgrade
+
+        msg = 'attempting to do database upgrade to version %s' % __dbversion__
+        print msg
+        api.upgrade(db_uri, repository_path, __dbversion__)
 
     def update_parser(self):
         self.parser.add_option('--sql',
                       action='store_true',
                       dest='just_sql',
                       help="Prints upgrade sql for further investigation",
                       default=False)

rhodecode/lib/dbmigrate/migrate/changeset/schema.py

@@ -1,669 +1,669
 """
    Schema module providing common schema operations.
 """
 import warnings
 
 from UserDict import DictMixin
 
 import sqlalchemy
 
 from sqlalchemy.schema import ForeignKeyConstraint
 from sqlalchemy.schema import UniqueConstraint
 
 from rhodecode.lib.dbmigrate.migrate.exceptions import *
 from rhodecode.lib.dbmigrate.migrate.changeset import SQLA_06
 from rhodecode.lib.dbmigrate.migrate.changeset.databases.visitor import (get_engine_visitor,
                                                  run_single_visitor)
 
 
 __all__ = [
     'create_column',
     'drop_column',
     'alter_column',
     'rename_table',
     'rename_index',
     'ChangesetTable',
     'ChangesetColumn',
     'ChangesetIndex',
     'ChangesetDefaultClause',
     'ColumnDelta',
 ]
 
 DEFAULT_ALTER_METADATA = True
 
 
 def create_column(column, table=None, *p, **kw):
     """Create a column, given the table.
     
     API to :meth:`ChangesetColumn.create`.
     """
     if table is not None:
         return table.create_column(column, *p, **kw)
     return column.create(*p, **kw)
 
 
 def drop_column(column, table=None, *p, **kw):
     """Drop a column, given the table.
     
     API to :meth:`ChangesetColumn.drop`.
     """
     if table is not None:
         return table.drop_column(column, *p, **kw)
     return column.drop(*p, **kw)
 
 
 def rename_table(table, name, engine=None, **kw):
     """Rename a table.
 
     If Table instance is given, engine is not used.
 
     API to :meth:`ChangesetTable.rename`.
 
     :param table: Table to be renamed.
     :param name: New name for Table.
     :param engine: Engine instance.
     :type table: string or Table instance
     :type name: string
     :type engine: obj
     """
     table = _to_table(table, engine)
     table.rename(name, **kw)
 
 
 def rename_index(index, name, table=None, engine=None, **kw):
     """Rename an index.
 
     If Index instance is given,
     table and engine are not used.
 
     API to :meth:`ChangesetIndex.rename`.
 
     :param index: Index to be renamed.
     :param name: New name for index.
     :param table: Table to which Index is reffered.
     :param engine: Engine instance.
     :type index: string or Index instance
     :type name: string
     :type table: string or Table instance
     :type engine: obj
     """
     index = _to_index(index, table, engine)
     index.rename(name, **kw)
 
 
 def alter_column(*p, **k):
     """Alter a column.
 
     This is a helper function that creates a :class:`ColumnDelta` and
     runs it.
 
     :argument column:
       The name of the column to be altered or a
       :class:`ChangesetColumn` column representing it.
 
     :param table:
       A :class:`~sqlalchemy.schema.Table` or table name to
       for the table where the column will be changed.
 
     :param engine:
       The :class:`~sqlalchemy.engine.base.Engine` to use for table
       reflection and schema alterations.
     
     :param alter_metadata:
       If `True`, which is the default, the
       :class:`~sqlalchemy.schema.Column` will also modified.
       If `False`, the :class:`~sqlalchemy.schema.Column` will be left
       as it was.
     
     :returns: A :class:`ColumnDelta` instance representing the change.
 
     
     """
-    
+
     k.setdefault('alter_metadata', DEFAULT_ALTER_METADATA)
 
     if 'table' not in k and isinstance(p[0], sqlalchemy.Column):
         k['table'] = p[0].table
     if 'engine' not in k:
         k['engine'] = k['table'].bind
 
     # deprecation
     if len(p) >= 2 and isinstance(p[1], sqlalchemy.Column):
         warnings.warn(
             "Passing a Column object to alter_column is deprecated."
             " Just pass in keyword parameters instead.",
             MigrateDeprecationWarning
             )
     engine = k['engine']
     delta = ColumnDelta(*p, **k)
 
     visitorcallable = get_engine_visitor(engine, 'schemachanger')
     engine._run_visitor(visitorcallable, delta)
 
     return delta
 
 
 def _to_table(table, engine=None):
     """Return if instance of Table, else construct new with metadata"""
     if isinstance(table, sqlalchemy.Table):
         return table
 
     # Given: table name, maybe an engine
     meta = sqlalchemy.MetaData()
     if engine is not None:
         meta.bind = engine
     return sqlalchemy.Table(table, meta)
 
 
 def _to_index(index, table=None, engine=None):
     """Return if instance of Index, else construct new with metadata"""
     if isinstance(index, sqlalchemy.Index):
         return index
 
     # Given: index name; table name required
     table = _to_table(table, engine)
     ret = sqlalchemy.Index(index)
     ret.table = table
     return ret
 
 
 class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem):
     """Extracts the differences between two columns/column-parameters
 
         May receive parameters arranged in several different ways:
 
         * **current_column, new_column, \*p, \*\*kw**
             Additional parameters can be specified to override column
             differences.
 
         * **current_column, \*p, \*\*kw**
             Additional parameters alter current_column. Table name is extracted
             from current_column object.
             Name is changed to current_column.name from current_name,
             if current_name is specified.
 
         * **current_col_name, \*p, \*\*kw**
             Table kw must specified.
 
         :param table: Table at which current Column should be bound to.\
         If table name is given, reflection will be used.
         :type table: string or Table instance
         :param alter_metadata: If True, it will apply changes to metadata.
         :type alter_metadata: bool
         :param metadata: If `alter_metadata` is true, \
         metadata is used to reflect table names into
         :type metadata: :class:`MetaData` instance
         :param engine: When reflecting tables, either engine or metadata must \
         be specified to acquire engine object.
         :type engine: :class:`Engine` instance
         :returns: :class:`ColumnDelta` instance provides interface for altered attributes to \
         `result_column` through :func:`dict` alike object.
 
         * :class:`ColumnDelta`.result_column is altered column with new attributes
 
         * :class:`ColumnDelta`.current_name is current name of column in db
 
 
     """
 
     # Column attributes that can be altered
     diff_keys = ('name', 'type', 'primary_key', 'nullable',
         'server_onupdate', 'server_default', 'autoincrement')
     diffs = dict()
     __visit_name__ = 'column'
 
     def __init__(self, *p, **kw):
         self.alter_metadata = kw.pop("alter_metadata", False)
         self.meta = kw.pop("metadata", None)
         self.engine = kw.pop("engine", None)
 
         # Things are initialized differently depending on how many column
         # parameters are given. Figure out how many and call the appropriate
         # method.
         if len(p) >= 1 and isinstance(p[0], sqlalchemy.Column):
             # At least one column specified
             if len(p) >= 2 and isinstance(p[1], sqlalchemy.Column):
                 # Two columns specified
                 diffs = self.compare_2_columns(*p, **kw)
             else:
                 # Exactly one column specified
                 diffs = self.compare_1_column(*p, **kw)
         else:
             # Zero columns specified
             if not len(p) or not isinstance(p[0], basestring):
                 raise ValueError("First argument must be column name")
             diffs = self.compare_parameters(*p, **kw)
 
         self.apply_diffs(diffs)
 
     def __repr__(self):
         return '<ColumnDelta altermetadata=%r, %s>' % (self.alter_metadata,
             super(ColumnDelta, self).__repr__())
 
     def __getitem__(self, key):
         if key not in self.keys():
             raise KeyError("No such diff key, available: %s" % self.diffs)
         return getattr(self.result_column, key)
 
     def __setitem__(self, key, value):
         if key not in self.keys():
             raise KeyError("No such diff key, available: %s" % self.diffs)
         setattr(self.result_column, key, value)
 
     def __delitem__(self, key):
         raise NotImplementedError
 
     def keys(self):
         return self.diffs.keys()
 
     def compare_parameters(self, current_name, *p, **k):
         """Compares Column objects with reflection"""
         self.table = k.pop('table')
         self.result_column = self._table.c.get(current_name)
         if len(p):
             k = self._extract_parameters(p, k, self.result_column)
         return k
 
     def compare_1_column(self, col, *p, **k):
         """Compares one Column object"""
         self.table = k.pop('table', None)
         if self.table is None:
             self.table = col.table
         self.result_column = col
         if len(p):
             k = self._extract_parameters(p, k, self.result_column)
         return k
 
     def compare_2_columns(self, old_col, new_col, *p, **k):
         """Compares two Column objects"""
         self.process_column(new_col)
         self.table = k.pop('table', None)
         # we cannot use bool() on table in SA06 
         if self.table is None:
             self.table = old_col.table
         if self.table is None:
             new_col.table
         self.result_column = old_col
 
         # set differences
         # leave out some stuff for later comp
         for key in (set(self.diff_keys) - set(('type',))):
             val = getattr(new_col, key, None)
             if getattr(self.result_column, key, None) != val:
                 k.setdefault(key, val)
 
         # inspect types
         if not self.are_column_types_eq(self.result_column.type, new_col.type):
             k.setdefault('type', new_col.type)
 
         if len(p):
             k = self._extract_parameters(p, k, self.result_column)
         return k
 
     def apply_diffs(self, diffs):
         """Populate dict and column object with new values"""
         self.diffs = diffs
         for key in self.diff_keys:
             if key in diffs:
                 setattr(self.result_column, key, diffs[key])
 
         self.process_column(self.result_column)
 
         # create an instance of class type if not yet
         if 'type' in diffs and callable(self.result_column.type):
             self.result_column.type = self.result_column.type()
 
         # add column to the table
         if self.table is not None and self.alter_metadata:
             self.result_column.add_to_table(self.table)
 
     def are_column_types_eq(self, old_type, new_type):
         """Compares two types to be equal"""
         ret = old_type.__class__ == new_type.__class__
 
         # String length is a special case
         if ret and isinstance(new_type, sqlalchemy.types.String):
             ret = (getattr(old_type, 'length', None) == \
                        getattr(new_type, 'length', None))
         return ret
 
     def _extract_parameters(self, p, k, column):
         """Extracts data from p and modifies diffs"""
         p = list(p)
         while len(p):
             if isinstance(p[0], basestring):
                 k.setdefault('name', p.pop(0))
             elif isinstance(p[0], sqlalchemy.types.AbstractType):
                 k.setdefault('type', p.pop(0))
             elif callable(p[0]):
                 p[0] = p[0]()
             else:
                 break
 
         if len(p):
             new_col = column.copy_fixed()
             new_col._init_items(*p)
             k = self.compare_2_columns(column, new_col, **k)
         return k
 
     def process_column(self, column):
         """Processes default values for column"""
         # XXX: this is a snippet from SA processing of positional parameters
         if not SQLA_06 and column.args:
             toinit = list(column.args)
         else:
             toinit = list()
 
         if column.server_default is not None:
             if isinstance(column.server_default, sqlalchemy.FetchedValue):
                 toinit.append(column.server_default)
             else:
                 toinit.append(sqlalchemy.DefaultClause(column.server_default))
         if column.server_onupdate is not None:
             if isinstance(column.server_onupdate, FetchedValue):
                 toinit.append(column.server_default)
             else:
                 toinit.append(sqlalchemy.DefaultClause(column.server_onupdate,
                                             for_update=True))
         if toinit:
             column._init_items(*toinit)
-            
+
         if not SQLA_06:
             column.args = []
 
     def _get_table(self):
         return getattr(self, '_table', None)
 
     def _set_table(self, table):
         if isinstance(table, basestring):
             if self.alter_metadata:
                 if not self.meta:
                     raise ValueError("metadata must be specified for table"
                         " reflection when using alter_metadata")
                 meta = self.meta
                 if self.engine:
                     meta.bind = self.engine
             else:
                 if not self.engine and not self.meta:
                     raise ValueError("engine or metadata must be specified"
                         " to reflect tables")
                 if not self.engine:
                     self.engine = self.meta.bind
                 meta = sqlalchemy.MetaData(bind=self.engine)
             self._table = sqlalchemy.Table(table, meta, autoload=True)
         elif isinstance(table, sqlalchemy.Table):
             self._table = table
             if not self.alter_metadata:
                 self._table.meta = sqlalchemy.MetaData(bind=self._table.bind)
 
     def _get_result_column(self):
         return getattr(self, '_result_column', None)
 
     def _set_result_column(self, column):
         """Set Column to Table based on alter_metadata evaluation."""
         self.process_column(column)
         if not hasattr(self, 'current_name'):
             self.current_name = column.name
         if self.alter_metadata:
             self._result_column = column
         else:
             self._result_column = column.copy_fixed()
 
     table = property(_get_table, _set_table)
     result_column = property(_get_result_column, _set_result_column)
 
 
 class ChangesetTable(object):
     """Changeset extensions to SQLAlchemy tables."""
 
     def create_column(self, column, *p, **kw):
         """Creates a column.
 
         The column parameter may be a column definition or the name of
         a column in this table.
 
         API to :meth:`ChangesetColumn.create`
 
         :param column: Column to be created
         :type column: Column instance or string
         """
         if not isinstance(column, sqlalchemy.Column):
             # It's a column name
             column = getattr(self.c, str(column))
         column.create(table=self, *p, **kw)
 
     def drop_column(self, column, *p, **kw):
         """Drop a column, given its name or definition.
 
         API to :meth:`ChangesetColumn.drop`
 
         :param column: Column to be droped
         :type column: Column instance or string
         """
         if not isinstance(column, sqlalchemy.Column):
             # It's a column name
             try:
                 column = getattr(self.c, str(column))
             except AttributeError:
                 # That column isn't part of the table. We don't need
                 # its entire definition to drop the column, just its
                 # name, so create a dummy column with the same name.
                 column = sqlalchemy.Column(str(column), sqlalchemy.Integer())
         column.drop(table=self, *p, **kw)
 
     def rename(self, name, connection=None, **kwargs):
         """Rename this table.
 
         :param name: New name of the table.
         :type name: string
         :param alter_metadata: If True, table will be removed from metadata
         :type alter_metadata: bool
         :param connection: reuse connection istead of creating new one.
         :type connection: :class:`sqlalchemy.engine.base.Connection` instance
         """
         self.alter_metadata = kwargs.pop('alter_metadata', DEFAULT_ALTER_METADATA)
         engine = self.bind
         self.new_name = name
         visitorcallable = get_engine_visitor(engine, 'schemachanger')
         run_single_visitor(engine, visitorcallable, self, connection, **kwargs)
 
         # Fix metadata registration
         if self.alter_metadata:
             self.name = name
             self.deregister()
             self._set_parent(self.metadata)
 
     def _meta_key(self):
         return sqlalchemy.schema._get_table_key(self.name, self.schema)
 
     def deregister(self):
         """Remove this table from its metadata"""
         key = self._meta_key()
         meta = self.metadata
         if key in meta.tables:
             del meta.tables[key]
 
 
 class ChangesetColumn(object):
     """Changeset extensions to SQLAlchemy columns."""
 
     def alter(self, *p, **k):
         """Makes a call to :func:`alter_column` for the column this
         method is called on. 
         """
         if 'table' not in k:
             k['table'] = self.table
         if 'engine' not in k:
             k['engine'] = k['table'].bind
         return alter_column(self, *p, **k)
 
     def create(self, table=None, index_name=None, unique_name=None,
                primary_key_name=None, populate_default=True, connection=None, **kwargs):
         """Create this column in the database.
 
         Assumes the given table exists. ``ALTER TABLE ADD COLUMN``,
         for most databases.
 
         :param table: Table instance to create on.
         :param index_name: Creates :class:`ChangesetIndex` on this column.
         :param unique_name: Creates :class:\
 `~migrate.changeset.constraint.UniqueConstraint` on this column.
         :param primary_key_name: Creates :class:\
 `~migrate.changeset.constraint.PrimaryKeyConstraint` on this column.
         :param alter_metadata: If True, column will be added to table object.
         :param populate_default: If True, created column will be \
 populated with defaults
         :param connection: reuse connection istead of creating new one.
         :type table: Table instance
         :type index_name: string
         :type unique_name: string
         :type primary_key_name: string
         :type alter_metadata: bool
         :type populate_default: bool
         :type connection: :class:`sqlalchemy.engine.base.Connection` instance
 
         :returns: self
         """
         self.populate_default = populate_default
         self.alter_metadata = kwargs.pop('alter_metadata', DEFAULT_ALTER_METADATA)
         self.index_name = index_name
         self.unique_name = unique_name
         self.primary_key_name = primary_key_name
         for cons in ('index_name', 'unique_name', 'primary_key_name'):
             self._check_sanity_constraints(cons)
 
         if self.alter_metadata:
             self.add_to_table(table)
         engine = self.table.bind
         visitorcallable = get_engine_visitor(engine, 'columngenerator')
         engine._run_visitor(visitorcallable, self, connection, **kwargs)
 
         # TODO: reuse existing connection
         if self.populate_default and self.default is not None:
             stmt = table.update().values({self: engine._execute_default(self.default)})
             engine.execute(stmt)
 
         return self
 
     def drop(self, table=None, connection=None, **kwargs):
         """Drop this column from the database, leaving its table intact.
 
         ``ALTER TABLE DROP COLUMN``, for most databases.
 
         :param alter_metadata: If True, column will be removed from table object.
         :type alter_metadata: bool
         :param connection: reuse connection istead of creating new one.
         :type connection: :class:`sqlalchemy.engine.base.Connection` instance
         """
         self.alter_metadata = kwargs.pop('alter_metadata', DEFAULT_ALTER_METADATA)
         if table is not None:
             self.table = table
         engine = self.table.bind
         if self.alter_metadata:
             self.remove_from_table(self.table, unset_table=False)
         visitorcallable = get_engine_visitor(engine, 'columndropper')
         engine._run_visitor(visitorcallable, self, connection, **kwargs)
         if self.alter_metadata:
             self.table = None
         return self
 
     def add_to_table(self, table):
         if table is not None  and self.table is None:
             self._set_parent(table)
 
     def _col_name_in_constraint(self, cons, name):
         return False
-    
+
     def remove_from_table(self, table, unset_table=True):
         # TODO: remove primary keys, constraints, etc
         if unset_table:
             self.table = None
-            
+
         to_drop = set()
         for index in table.indexes:
             columns = []
             for col in index.columns:
                 if col.name != self.name:
                     columns.append(col)
             if columns:
                 index.columns = columns
             else:
                 to_drop.add(index)
         table.indexes = table.indexes - to_drop
-        
+
         to_drop = set()
         for cons in table.constraints:
             # TODO: deal with other types of constraint
             if isinstance(cons, (ForeignKeyConstraint,
                                 UniqueConstraint)):
                 for col_name in cons.columns:
                     if not isinstance(col_name, basestring):
                         col_name = col_name.name
                     if self.name == col_name:
                         to_drop.add(cons)
         table.constraints = table.constraints - to_drop
-        
+
         if table.c.contains_column(self):
             table.c.remove(self)
 
     # TODO: this is fixed in 0.6
     def copy_fixed(self, **kw):
         """Create a copy of this ``Column``, with all attributes."""
         return sqlalchemy.Column(self.name, self.type, self.default,
             key=self.key,
             primary_key=self.primary_key,
             nullable=self.nullable,
             quote=self.quote,
             index=self.index,
             unique=self.unique,
             onupdate=self.onupdate,
             autoincrement=self.autoincrement,
             server_default=self.server_default,
             server_onupdate=self.server_onupdate,
             *[c.copy(**kw) for c in self.constraints])
 
     def _check_sanity_constraints(self, name):
         """Check if constraints names are correct"""
         obj = getattr(self, name)
         if (getattr(self, name[:-5]) and not obj):
             raise InvalidConstraintError("Column.create() accepts index_name,"
             " primary_key_name and unique_name to generate constraints")
         if not isinstance(obj, basestring) and obj is not None:
             raise InvalidConstraintError(
             "%s argument for column must be constraint name" % name)
 
 
 class ChangesetIndex(object):
     """Changeset extensions to SQLAlchemy Indexes."""
 
     __visit_name__ = 'index'
 
     def rename(self, name, connection=None, **kwargs):
         """Change the name of an index.
 
         :param name: New name of the Index.
         :type name: string
         :param alter_metadata: If True, Index object will be altered.
         :type alter_metadata: bool
         :param connection: reuse connection istead of creating new one.
         :type connection: :class:`sqlalchemy.engine.base.Connection` instance
         """
         self.alter_metadata = kwargs.pop('alter_metadata', DEFAULT_ALTER_METADATA)
         engine = self.table.bind
         self.new_name = name
         visitorcallable = get_engine_visitor(engine, 'schemachanger')
         engine._run_visitor(visitorcallable, self, connection, **kwargs)
         if self.alter_metadata:
             self.name = name
 
 
 class ChangesetDefaultClause(object):
     """Implements comparison between :class:`DefaultClause` instances"""
 
     def __eq__(self, other):
         if isinstance(other, self.__class__):
             if self.arg == other.arg:
                 return True
 
     def __ne__(self, other):
         return not self.__eq__(other)

rhodecode/lib/dbmigrate/migrate/versioning/api.py

@@ -1,383 +1,383
 """
    This module provides an external API to the versioning system.
 
    .. versionchanged:: 0.6.0
     :func:`migrate.versioning.api.test` and schema diff functions
     changed order of positional arguments so all accept `url` and `repository`
     as first arguments.
 
    .. versionchanged:: 0.5.4
     ``--preview_sql`` displays source file when using SQL scripts.
     If Python script is used, it runs the action with mocked engine and
     returns captured SQL statements.
 
    .. versionchanged:: 0.5.4
     Deprecated ``--echo`` parameter in favour of new
     :func:`migrate.versioning.util.construct_engine` behavior.
 """
 
 # Dear migrate developers,
 #
 # please do not comment this module using sphinx syntax because its
 # docstrings are presented as user help and most users cannot
 # interpret sphinx annotated ReStructuredText.
 #
 # Thanks,
 # Jan Dittberner
 
 import sys
 import inspect
 import logging
 
 from rhodecode.lib.dbmigrate.migrate import exceptions
-from rhodecode.lib.dbmigrate.migrate.versioning import (repository, schema, version,
-    script as script_) # command name conflict
+from rhodecode.lib.dbmigrate.migrate.versioning import repository, schema, version, \
+    script as script_ # command name conflict
 from rhodecode.lib.dbmigrate.migrate.versioning.util import catch_known_errors, with_engine
 
 
 log = logging.getLogger(__name__)
 command_desc = {
     'help': 'displays help on a given command',
     'create': 'create an empty repository at the specified path',
     'script': 'create an empty change Python script',
     'script_sql': 'create empty change SQL scripts for given database',
     'version': 'display the latest version available in a repository',
     'db_version': 'show the current version of the repository under version control',
     'source': 'display the Python code for a particular version in this repository',
     'version_control': 'mark a database as under this repository\'s version control',
     'upgrade': 'upgrade a database to a later version',
     'downgrade': 'downgrade a database to an earlier version',
     'drop_version_control': 'removes version control from a database',
     'manage': 'creates a Python script that runs Migrate with a set of default values',
     'test': 'performs the upgrade and downgrade command on the given database',
     'compare_model_to_db': 'compare MetaData against the current database state',
     'create_model': 'dump the current database as a Python model to stdout',
     'make_update_script_for_model': 'create a script changing the old MetaData to the new (current) MetaData',
     'update_db_from_model': 'modify the database to match the structure of the current MetaData',
 }
 __all__ = command_desc.keys()
 
 Repository = repository.Repository
 ControlledSchema = schema.ControlledSchema
 VerNum = version.VerNum
 PythonScript = script_.PythonScript
 SqlScript = script_.SqlScript
 
 
 # deprecated
 def help(cmd=None, **opts):
     """%prog help COMMAND
 
     Displays help on a given command.
     """
     if cmd is None:
         raise exceptions.UsageError(None)
     try:
         func = globals()[cmd]
     except:
         raise exceptions.UsageError(
             "'%s' isn't a valid command. Try 'help COMMAND'" % cmd)
     ret = func.__doc__
     if sys.argv[0]:
         ret = ret.replace('%prog', sys.argv[0])
     return ret
 
 @catch_known_errors
 def create(repository, name, **opts):
     """%prog create REPOSITORY_PATH NAME [--table=TABLE]
 
     Create an empty repository at the specified path.
 
     You can specify the version_table to be used; by default, it is
     'migrate_version'.  This table is created in all version-controlled
     databases.
     """
     repo_path = Repository.create(repository, name, **opts)
 
 
 @catch_known_errors
 def script(description, repository, **opts):
     """%prog script DESCRIPTION REPOSITORY_PATH
 
     Create an empty change script using the next unused version number
     appended with the given description.
 
     For instance, manage.py script "Add initial tables" creates:
     repository/versions/001_Add_initial_tables.py
     """
     repo = Repository(repository)
     repo.create_script(description, **opts)
 
 
 @catch_known_errors
 def script_sql(database, repository, **opts):
     """%prog script_sql DATABASE REPOSITORY_PATH
 
     Create empty change SQL scripts for given DATABASE, where DATABASE
     is either specific ('postgres', 'mysql', 'oracle', 'sqlite', etc.)
     or generic ('default').
 
     For instance, manage.py script_sql postgres creates:
     repository/versions/001_postgres_upgrade.sql and
     repository/versions/001_postgres_postgres.sql
     """
     repo = Repository(repository)
     repo.create_script_sql(database, **opts)
 
 
 def version(repository, **opts):
     """%prog version REPOSITORY_PATH
 
     Display the latest version available in a repository.
     """
     repo = Repository(repository)
     return repo.latest
 
 
 @with_engine
 def db_version(url, repository, **opts):
     """%prog db_version URL REPOSITORY_PATH
 
     Show the current version of the repository with the given
     connection string, under version control of the specified
     repository.
 
     The url should be any valid SQLAlchemy connection string.
     """
     engine = opts.pop('engine')
     schema = ControlledSchema(engine, repository)
     return schema.version
 
 
 def source(version, dest=None, repository=None, **opts):
     """%prog source VERSION [DESTINATION] --repository=REPOSITORY_PATH
 
     Display the Python code for a particular version in this
     repository.  Save it to the file at DESTINATION or, if omitted,
     send to stdout.
     """
     if repository is None:
         raise exceptions.UsageError("A repository must be specified")
     repo = Repository(repository)
     ret = repo.version(version).script().source()
     if dest is not None:
         dest = open(dest, 'w')
         dest.write(ret)
         dest.close()
         ret = None
     return ret
 
 
 def upgrade(url, repository, version=None, **opts):
     """%prog upgrade URL REPOSITORY_PATH [VERSION] [--preview_py|--preview_sql]
 
     Upgrade a database to a later version.
 
     This runs the upgrade() function defined in your change scripts.
 
     By default, the database is updated to the latest available
     version. You may specify a version instead, if you wish.
 
     You may preview the Python or SQL code to be executed, rather than
     actually executing it, using the appropriate 'preview' option.
     """
     err = "Cannot upgrade a database of version %s to version %s. "\
         "Try 'downgrade' instead."
     return _migrate(url, repository, version, upgrade=True, err=err, **opts)
 
 
 def downgrade(url, repository, version, **opts):
     """%prog downgrade URL REPOSITORY_PATH VERSION [--preview_py|--preview_sql]
 
     Downgrade a database to an earlier version.
 
     This is the reverse of upgrade; this runs the downgrade() function
     defined in your change scripts.
 
     You may preview the Python or SQL code to be executed, rather than
     actually executing it, using the appropriate 'preview' option.
     """
     err = "Cannot downgrade a database of version %s to version %s. "\
         "Try 'upgrade' instead."
     return _migrate(url, repository, version, upgrade=False, err=err, **opts)
 
 @with_engine
 def test(url, repository, **opts):
     """%prog test URL REPOSITORY_PATH [VERSION]
 
     Performs the upgrade and downgrade option on the given
     database. This is not a real test and may leave the database in a
     bad state. You should therefore better run the test on a copy of
     your database.
     """
     engine = opts.pop('engine')
     repos = Repository(repository)
     script = repos.version(None).script()
 
     # Upgrade
     log.info("Upgrading...")
     script.run(engine, 1)
     log.info("done")
 
     log.info("Downgrading...")
     script.run(engine, -1)
     log.info("done")
     log.info("Success")
 
 
 @with_engine
 def version_control(url, repository, version=None, **opts):
     """%prog version_control URL REPOSITORY_PATH [VERSION]
 
     Mark a database as under this repository's version control.
 
     Once a database is under version control, schema changes should
     only be done via change scripts in this repository.
 
     This creates the table version_table in the database.
 
     The url should be any valid SQLAlchemy connection string.
 
     By default, the database begins at version 0 and is assumed to be
     empty.  If the database is not empty, you may specify a version at
     which to begin instead. No attempt is made to verify this
     version's correctness - the database schema is expected to be
     identical to what it would be if the database were created from
     scratch.
     """
     engine = opts.pop('engine')
     ControlledSchema.create(engine, repository, version)
 
 
 @with_engine
 def drop_version_control(url, repository, **opts):
     """%prog drop_version_control URL REPOSITORY_PATH
 
     Removes version control from a database.
     """
     engine = opts.pop('engine')
     schema = ControlledSchema(engine, repository)
     schema.drop()
 
 
 def manage(file, **opts):
     """%prog manage FILENAME [VARIABLES...]
 
     Creates a script that runs Migrate with a set of default values.
 
     For example::
 
         %prog manage manage.py --repository=/path/to/repository \
 --url=sqlite:///project.db
 
     would create the script manage.py. The following two commands
     would then have exactly the same results::
 
         python manage.py version
         %prog version --repository=/path/to/repository
     """
     Repository.create_manage_file(file, **opts)
 
 
 @with_engine
 def compare_model_to_db(url, repository, model, **opts):
     """%prog compare_model_to_db URL REPOSITORY_PATH MODEL
 
     Compare the current model (assumed to be a module level variable
     of type sqlalchemy.MetaData) against the current database.
 
     NOTE: This is EXPERIMENTAL.
     """  # TODO: get rid of EXPERIMENTAL label
     engine = opts.pop('engine')
     return ControlledSchema.compare_model_to_db(engine, model, repository)
 
 
 @with_engine
 def create_model(url, repository, **opts):
     """%prog create_model URL REPOSITORY_PATH [DECLERATIVE=True]
 
     Dump the current database as a Python model to stdout.
 
     NOTE: This is EXPERIMENTAL.
     """  # TODO: get rid of EXPERIMENTAL label
     engine = opts.pop('engine')
     declarative = opts.get('declarative', False)
     return ControlledSchema.create_model(engine, repository, declarative)
 
 
 @catch_known_errors
 @with_engine
 def make_update_script_for_model(url, repository, oldmodel, model, **opts):
     """%prog make_update_script_for_model URL OLDMODEL MODEL REPOSITORY_PATH
 
     Create a script changing the old Python model to the new (current)
     Python model, sending to stdout.
 
     NOTE: This is EXPERIMENTAL.
     """  # TODO: get rid of EXPERIMENTAL label
     engine = opts.pop('engine')
     return PythonScript.make_update_script_for_model(
         engine, oldmodel, model, repository, **opts)
 
 
 @with_engine
 def update_db_from_model(url, repository, model, **opts):
     """%prog update_db_from_model URL REPOSITORY_PATH MODEL
 
     Modify the database to match the structure of the current Python
     model. This also sets the db_version number to the latest in the
     repository.
 
     NOTE: This is EXPERIMENTAL.
     """  # TODO: get rid of EXPERIMENTAL label
     engine = opts.pop('engine')
     schema = ControlledSchema(engine, repository)
     schema.update_db_from_model(model)
 
 @with_engine
 def _migrate(url, repository, version, upgrade, err, **opts):
     engine = opts.pop('engine')
     url = str(engine.url)
     schema = ControlledSchema(engine, repository)
     version = _migrate_version(schema, version, upgrade, err)
 
     changeset = schema.changeset(version)
     for ver, change in changeset:
         nextver = ver + changeset.step
         log.info('%s -> %s... ', ver, nextver)
 
         if opts.get('preview_sql'):
             if isinstance(change, PythonScript):
                 log.info(change.preview_sql(url, changeset.step, **opts))
             elif isinstance(change, SqlScript):
                 log.info(change.source())
 
         elif opts.get('preview_py'):
             if not isinstance(change, PythonScript):
                 raise exceptions.UsageError("Python source can be only displayed"
                     " for python migration files")
             source_ver = max(ver, nextver)
             module = schema.repository.version(source_ver).script().module
             funcname = upgrade and "upgrade" or "downgrade"
             func = getattr(module, funcname)
             log.info(inspect.getsource(func))
         else:
             schema.runchange(ver, change, changeset.step)
             log.info('done')
 
 
 def _migrate_version(schema, version, upgrade, err):
     if version is None:
         return version
     # Version is specified: ensure we're upgrading in the right direction
     # (current version < target version for upgrading; reverse for down)
     version = VerNum(version)
     cur = schema.version
     if upgrade is not None:
         if upgrade:
             direction = cur <= version
         else:
             direction = cur >= version
         if not direction:
             raise exceptions.KnownError(err % (cur, version))
     return version

rhodecode/lib/dbmigrate/migrate/versioning/script/sql.py

@@ -1,49 +1,48
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 import logging
 import shutil
 
 from rhodecode.lib.dbmigrate.migrate.versioning.script import base
 from rhodecode.lib.dbmigrate.migrate.versioning.template import Template
 
 
 log = logging.getLogger(__name__)
 
 class SqlScript(base.BaseScript):
     """A file containing plain SQL statements."""
 
     @classmethod
     def create(cls, path, **opts):
         """Create an empty migration script at specified path
         
         :returns: :class:`SqlScript instance <migrate.versioning.script.sql.SqlScript>`"""
         cls.require_notfound(path)
-
         src = Template(opts.pop('templates_path', None)).get_sql_script(theme=opts.pop('templates_theme', None))
         shutil.copy(src, path)
         return cls(path)
 
     # TODO: why is step parameter even here?
     def run(self, engine, step=None, executemany=True):
         """Runs SQL script through raw dbapi execute call"""
         text = self.source()
         # Don't rely on SA's autocommit here
         # (SA uses .startswith to check if a commit is needed. What if script
         # starts with a comment?)
         conn = engine.connect()
         try:
             trans = conn.begin()
             try:
                 # HACK: SQLite doesn't allow multiple statements through
                 # its execute() method, but it provides executescript() instead
                 dbapi = conn.engine.raw_connection()
                 if executemany and getattr(dbapi, 'executescript', None):
                     dbapi.executescript(text)
                 else:
                     conn.execute(text)
                 trans.commit()
             except:
                 trans.rollback()
                 raise
         finally:
             conn.close()

rhodecode/lib/dbmigrate/migrate/versioning/template.py

@@ -1,94 +1,94
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 import os
 import shutil
 import sys
 
 from pkg_resources import resource_filename
 
 from rhodecode.lib.dbmigrate.migrate.versioning.config import *
 from rhodecode.lib.dbmigrate.migrate.versioning import pathed
 
 
 class Collection(pathed.Pathed):
     """A collection of templates of a specific type"""
     _mask = None
 
     def get_path(self, file):
         return os.path.join(self.path, str(file))
 
 
 class RepositoryCollection(Collection):
     _mask = '%s'
 
 class ScriptCollection(Collection):
     _mask = '%s.py_tmpl'
 
 class ManageCollection(Collection):
     _mask = '%s.py_tmpl'
 
 class SQLScriptCollection(Collection):
     _mask = '%s.py_tmpl'
 
 class Template(pathed.Pathed):
     """Finds the paths/packages of various Migrate templates.
     
     :param path: Templates are loaded from rhodecode.lib.dbmigrate.migrate package
     if `path` is not provided.
     """
-    pkg = 'migrate.versioning.templates'
+    pkg = 'rhodecode.lib.dbmigrate.migrate.versioning.templates'
     _manage = 'manage.py_tmpl'
 
     def __new__(cls, path=None):
         if path is None:
             path = cls._find_path(cls.pkg)
         return super(Template, cls).__new__(cls, path)
 
     def __init__(self, path=None):
         if path is None:
             path = Template._find_path(self.pkg)
         super(Template, self).__init__(path)
         self.repository = RepositoryCollection(os.path.join(path, 'repository'))
         self.script = ScriptCollection(os.path.join(path, 'script'))
         self.manage = ManageCollection(os.path.join(path, 'manage'))
         self.sql_script = SQLScriptCollection(os.path.join(path, 'sql_script'))
 
     @classmethod
     def _find_path(cls, pkg):
         """Returns absolute path to dotted python package."""
         tmp_pkg = pkg.rsplit('.', 1)
 
         if len(tmp_pkg) != 1:
             return resource_filename(tmp_pkg[0], tmp_pkg[1])
         else:
             return resource_filename(tmp_pkg[0], '')
 
     def _get_item(self, collection, theme=None):
         """Locates and returns collection.
         
         :param collection: name of collection to locate
         :param type_: type of subfolder in collection (defaults to "_default")
         :returns: (package, source)
         :rtype: str, str
         """
         item = getattr(self, collection)
         theme_mask = getattr(item, '_mask')
         theme = theme_mask % (theme or 'default')
         return item.get_path(theme)
 
     def get_repository(self, *a, **kw):
         """Calls self._get_item('repository', *a, **kw)"""
         return self._get_item('repository', *a, **kw)
-    
+
     def get_script(self, *a, **kw):
         """Calls self._get_item('script', *a, **kw)"""
         return self._get_item('script', *a, **kw)
 
     def get_sql_script(self, *a, **kw):
         """Calls self._get_item('sql_script', *a, **kw)"""
         return self._get_item('sql_script', *a, **kw)
 
     def get_manage(self, *a, **kw):
         """Calls self._get_item('manage', *a, **kw)"""
         return self._get_item('manage', *a, **kw)

rhodecode/lib/dbmigrate/versions/001_initial_release.py

@@ -1,238 +1,237
-from migrate import *
-
 #==============================================================================
 # DB INITIAL MODEL
 #==============================================================================
 import logging
 import datetime
 
 from sqlalchemy import *
 from sqlalchemy.exc import DatabaseError
 from sqlalchemy.orm import relation, backref, class_mapper
 from sqlalchemy.orm.session import Session
+from rhodecode.model.meta import Base
 
-from rhodecode.model.meta import Base
+from rhodecode.lib.dbmigrate.migrate import *
 
 log = logging.getLogger(__name__)
 
 class BaseModel(object):
 
     @classmethod
     def _get_keys(cls):
         """return column names for this model """
         return class_mapper(cls).c.keys()
 
     def get_dict(self):
         """return dict with keys and values corresponding 
         to this model data """
 
         d = {}
         for k in self._get_keys():
             d[k] = getattr(self, k)
         return d
 
     def get_appstruct(self):
         """return list with keys and values tupples corresponding 
         to this model data """
 
         l = []
         for k in self._get_keys():
             l.append((k, getattr(self, k),))
         return l
 
     def populate_obj(self, populate_dict):
         """populate model with data from given populate_dict"""
 
         for k in self._get_keys():
             if k in populate_dict:
                 setattr(self, k, populate_dict[k])
 
 class RhodeCodeSettings(Base, BaseModel):
     __tablename__ = 'rhodecode_settings'
     __table_args__ = (UniqueConstraint('app_settings_name'), {'useexisting':True})
     app_settings_id = Column("app_settings_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     app_settings_name = Column("app_settings_name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     app_settings_value = Column("app_settings_value", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
     def __init__(self, k, v):
         self.app_settings_name = k
         self.app_settings_value = v
 
     def __repr__(self):
         return "<RhodeCodeSetting('%s:%s')>" % (self.app_settings_name,
                                                 self.app_settings_value)
 
 class RhodeCodeUi(Base, BaseModel):
     __tablename__ = 'rhodecode_ui'
     __table_args__ = {'useexisting':True}
     ui_id = Column("ui_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     ui_section = Column("ui_section", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_key = Column("ui_key", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_value = Column("ui_value", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_active = Column("ui_active", Boolean(), nullable=True, unique=None, default=True)
 
 
 class User(Base, BaseModel):
     __tablename__ = 'users'
     __table_args__ = (UniqueConstraint('username'), UniqueConstraint('email'), {'useexisting':True})
     user_id = Column("user_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     username = Column("username", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     password = Column("password", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     active = Column("active", Boolean(), nullable=True, unique=None, default=None)
     admin = Column("admin", Boolean(), nullable=True, unique=None, default=False)
     name = Column("name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     lastname = Column("lastname", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     email = Column("email", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     last_login = Column("last_login", DateTime(timezone=False), nullable=True, unique=None, default=None)
     is_ldap = Column("is_ldap", Boolean(), nullable=False, unique=None, default=False)
 
     user_log = relation('UserLog', cascade='all')
     user_perms = relation('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id", cascade='all')
 
     repositories = relation('Repository')
     user_followers = relation('UserFollowing', primaryjoin='UserFollowing.follows_user_id==User.user_id', cascade='all')
 
     @property
     def full_contact(self):
         return '%s %s <%s>' % (self.name, self.lastname, self.email)
 
     def __repr__(self):
         return "<User('id:%s:%s')>" % (self.user_id, self.username)
 
     def update_lastlogin(self):
         """Update user lastlogin"""
 
         try:
             session = Session.object_session(self)
             self.last_login = datetime.datetime.now()
             session.add(self)
             session.commit()
             log.debug('updated user %s lastlogin', self.username)
         except (DatabaseError,):
             session.rollback()
 
 
 class UserLog(Base, BaseModel):
     __tablename__ = 'user_logs'
     __table_args__ = {'useexisting':True}
     user_log_id = Column("user_log_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
     repository_id = Column("repository_id", Integer(length=None, convert_unicode=False, assert_unicode=None), ForeignKey(u'repositories.repo_id'), nullable=False, unique=None, default=None)
     repository_name = Column("repository_name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     user_ip = Column("user_ip", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     action = Column("action", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     action_date = Column("action_date", DateTime(timezone=False), nullable=True, unique=None, default=None)
 
     user = relation('User')
     repository = relation('Repository')
 
 class Repository(Base, BaseModel):
     __tablename__ = 'repositories'
     __table_args__ = (UniqueConstraint('repo_name'), {'useexisting':True},)
     repo_id = Column("repo_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     repo_name = Column("repo_name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
     repo_type = Column("repo_type", String(length=None, convert_unicode=False, assert_unicode=None), nullable=False, unique=False, default=None)
     user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=False, default=None)
     private = Column("private", Boolean(), nullable=True, unique=None, default=None)
     enable_statistics = Column("statistics", Boolean(), nullable=True, unique=None, default=True)
     description = Column("description", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     fork_id = Column("fork_id", Integer(), ForeignKey(u'repositories.repo_id'), nullable=True, unique=False, default=None)
 
     user = relation('User')
     fork = relation('Repository', remote_side=repo_id)
     repo_to_perm = relation('RepoToPerm', cascade='all')
     stats = relation('Statistics', cascade='all', uselist=False)
 
     repo_followers = relation('UserFollowing', primaryjoin='UserFollowing.follows_repo_id==Repository.repo_id', cascade='all')
 
 
     def __repr__(self):
         return "<Repository('%s:%s')>" % (self.repo_id, self.repo_name)
 
 class Permission(Base, BaseModel):
     __tablename__ = 'permissions'
     __table_args__ = {'useexisting':True}
     permission_id = Column("permission_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     permission_name = Column("permission_name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     permission_longname = Column("permission_longname", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
     def __repr__(self):
         return "<Permission('%s:%s')>" % (self.permission_id, self.permission_name)
 
 class RepoToPerm(Base, BaseModel):
     __tablename__ = 'repo_to_perm'
     __table_args__ = (UniqueConstraint('user_id', 'repository_id'), {'useexisting':True})
     repo_to_perm_id = Column("repo_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey(u'permissions.permission_id'), nullable=False, unique=None, default=None)
     repository_id = Column("repository_id", Integer(), ForeignKey(u'repositories.repo_id'), nullable=False, unique=None, default=None)
 
     user = relation('User')
     permission = relation('Permission')
     repository = relation('Repository')
 
 class UserToPerm(Base, BaseModel):
     __tablename__ = 'user_to_perm'
     __table_args__ = (UniqueConstraint('user_id', 'permission_id'), {'useexisting':True})
     user_to_perm_id = Column("user_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey(u'permissions.permission_id'), nullable=False, unique=None, default=None)
 
     user = relation('User')
     permission = relation('Permission')
 
 class Statistics(Base, BaseModel):
     __tablename__ = 'statistics'
     __table_args__ = (UniqueConstraint('repository_id'), {'useexisting':True})
     stat_id = Column("stat_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     repository_id = Column("repository_id", Integer(), ForeignKey(u'repositories.repo_id'), nullable=False, unique=True, default=None)
     stat_on_revision = Column("stat_on_revision", Integer(), nullable=False)
     commit_activity = Column("commit_activity", LargeBinary(), nullable=False)#JSON data
     commit_activity_combined = Column("commit_activity_combined", LargeBinary(), nullable=False)#JSON data
     languages = Column("languages", LargeBinary(), nullable=False)#JSON data
 
     repository = relation('Repository', single_parent=True)
 
 class UserFollowing(Base, BaseModel):
     __tablename__ = 'user_followings'
     __table_args__ = (UniqueConstraint('user_id', 'follows_repository_id'),
                       UniqueConstraint('user_id', 'follows_user_id')
                       , {'useexisting':True})
 
     user_following_id = Column("user_following_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
     follows_repo_id = Column("follows_repository_id", Integer(), ForeignKey(u'repositories.repo_id'), nullable=True, unique=None, default=None)
     follows_user_id = Column("follows_user_id", Integer(), ForeignKey(u'users.user_id'), nullable=True, unique=None, default=None)
 
     user = relation('User', primaryjoin='User.user_id==UserFollowing.user_id')
 
     follows_user = relation('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')
     follows_repository = relation('Repository')
 
 
 class CacheInvalidation(Base, BaseModel):
     __tablename__ = 'cache_invalidation'
     __table_args__ = (UniqueConstraint('cache_key'), {'useexisting':True})
     cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     cache_key = Column("cache_key", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     cache_args = Column("cache_args", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False)
 
 
     def __init__(self, cache_key, cache_args=''):
         self.cache_key = cache_key
         self.cache_args = cache_args
         self.cache_active = False
 
     def __repr__(self):
         return "<CacheInvalidation('%s:%s')>" % (self.cache_id, self.cache_key)
 
 
 def upgrade(migrate_engine):
     # Upgrade operations go here. Don't create your own engine; bind migrate_engine
     # to your metadata
     Base.metadata.create_all(bind=migrate_engine, checkfirst=False)
 
 def downgrade(migrate_engine):
     # Operations to reverse the above upgrade go here.
     Base.metadata.drop_all(bind=migrate_engine, checkfirst=False)

rhodecode/lib/dbmigrate/versions/002_version_1_1_0.py

@@ -1,118 +1,129
-from sqlalchemy import *
-from sqlalchemy.orm import relation
+import logging
+import datetime
 
-from migrate import *
-from migrate.changeset import *
-from rhodecode.model.meta import Base, BaseModel
+from sqlalchemy import *
+from sqlalchemy.exc import DatabaseError
+from sqlalchemy.orm import relation, backref, class_mapper
+from sqlalchemy.orm.session import Session
+from rhodecode.model.meta import Base
+from rhodecode.model.db import BaseModel
+
+from rhodecode.lib.dbmigrate.migrate import *
+
+log = logging.getLogger(__name__)
 
 def upgrade(migrate_engine):
     """ Upgrade operations go here. 
     Don't create your own engine; bind migrate_engine to your metadata
     """
 
     #==========================================================================
     # Upgrade of `users` table
     #==========================================================================
     tblname = 'users'
     tbl = Table(tblname, MetaData(bind=migrate_engine), autoload=True,
                     autoload_with=migrate_engine)
 
     #ADD is_ldap column
-    is_ldap = Column("is_ldap", Boolean(), nullable=False,
+    is_ldap = Column("is_ldap", Boolean(), nullable=True,
                      unique=None, default=False)
-    is_ldap.create(tbl)
-
+    is_ldap.create(tbl, populate_default=True)
+    is_ldap.alter(nullable=False)
 
     #==========================================================================
     # Upgrade of `user_logs` table
     #==========================================================================    
 
     tblname = 'users'
     tbl = Table(tblname, MetaData(bind=migrate_engine), autoload=True,
                     autoload_with=migrate_engine)
 
     #ADD revision column
     revision = Column('revision', TEXT(length=None, convert_unicode=False,
                                        assert_unicode=None),
                       nullable=True, unique=None, default=None)
     revision.create(tbl)
 
 
 
     #==========================================================================
     # Upgrade of `repositories` table
     #==========================================================================    
     tblname = 'users'
     tbl = Table(tblname, MetaData(bind=migrate_engine), autoload=True,
                     autoload_with=migrate_engine)
 
     #ADD repo_type column
     repo_type = Column("repo_type", String(length=None, convert_unicode=False,
                                            assert_unicode=None),
-                       nullable=False, unique=False, default=None)
-    repo_type.create(tbl)
+                       nullable=True, unique=False, default='hg')
 
+    repo_type.create(tbl, populate_default=True)
+    repo_type.alter(nullable=False)
 
     #ADD statistics column
     enable_statistics = Column("statistics", Boolean(), nullable=True,
                                unique=None, default=True)
     enable_statistics.create(tbl)
 
 
 
     #==========================================================================
     # Add table `user_followings`
     #==========================================================================
     tblname = 'user_followings'
+
     class UserFollowing(Base, BaseModel):
         __tablename__ = 'user_followings'
         __table_args__ = (UniqueConstraint('user_id', 'follows_repository_id'),
                           UniqueConstraint('user_id', 'follows_user_id')
                           , {'useexisting':True})
 
         user_following_id = Column("user_following_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
         user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
         follows_repo_id = Column("follows_repository_id", Integer(), ForeignKey(u'repositories.repo_id'), nullable=True, unique=None, default=None)
         follows_user_id = Column("follows_user_id", Integer(), ForeignKey(u'users.user_id'), nullable=True, unique=None, default=None)
 
         user = relation('User', primaryjoin='User.user_id==UserFollowing.user_id')
 
         follows_user = relation('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')
         follows_repository = relation('Repository')
 
     Base.metadata.tables[tblname].create(migrate_engine)
 
     #==========================================================================
     # Add table `cache_invalidation`
     #==========================================================================
+    tblname = 'cache_invalidation'
+
     class CacheInvalidation(Base, BaseModel):
         __tablename__ = 'cache_invalidation'
         __table_args__ = (UniqueConstraint('cache_key'), {'useexisting':True})
         cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
         cache_key = Column("cache_key", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
         cache_args = Column("cache_args", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
         cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False)
 
 
         def __init__(self, cache_key, cache_args=''):
             self.cache_key = cache_key
             self.cache_args = cache_args
             self.cache_active = False
 
         def __repr__(self):
             return "<CacheInvalidation('%s:%s')>" % (self.cache_id, self.cache_key)
 
     Base.metadata.tables[tblname].create(migrate_engine)
 
     return
 
 
 
 
 
 
 def downgrade(migrate_engine):
     meta = MetaData()
     meta.bind = migrate_engine
 
 

rhodecode/model/db.py

@@ -1,257 +1,257
 # -*- coding: utf-8 -*-
 """
     rhodecode.model.db
     ~~~~~~~~~~~~~~~~~~
     
     Database Models for RhodeCode
     
     :created_on: Apr 08, 2010
     :author: marcink
     :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 import logging
 import datetime
 
 from sqlalchemy import *
 from sqlalchemy.exc import DatabaseError
 from sqlalchemy.orm import relation, backref, class_mapper
 from sqlalchemy.orm.session import Session
 
 from rhodecode.model.meta import Base
 
 log = logging.getLogger(__name__)
 
 class BaseModel(object):
 
     @classmethod
     def _get_keys(cls):
         """return column names for this model """
         return class_mapper(cls).c.keys()
 
     def get_dict(self):
         """return dict with keys and values corresponding 
         to this model data """
 
         d = {}
         for k in self._get_keys():
             d[k] = getattr(self, k)
         return d
 
     def get_appstruct(self):
         """return list with keys and values tupples corresponding 
         to this model data """
 
         l = []
         for k in self._get_keys():
             l.append((k, getattr(self, k),))
         return l
 
     def populate_obj(self, populate_dict):
         """populate model with data from given populate_dict"""
 
         for k in self._get_keys():
             if k in populate_dict:
                 setattr(self, k, populate_dict[k])
 
 class RhodeCodeSettings(Base, BaseModel):
     __tablename__ = 'rhodecode_settings'
     __table_args__ = (UniqueConstraint('app_settings_name'), {'useexisting':True})
     app_settings_id = Column("app_settings_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     app_settings_name = Column("app_settings_name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     app_settings_value = Column("app_settings_value", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
     def __init__(self, k, v):
         self.app_settings_name = k
         self.app_settings_value = v
 
     def __repr__(self):
         return "<RhodeCodeSetting('%s:%s')>" % (self.app_settings_name,
                                                 self.app_settings_value)
 
 class RhodeCodeUi(Base, BaseModel):
     __tablename__ = 'rhodecode_ui'
     __table_args__ = {'useexisting':True}
     ui_id = Column("ui_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     ui_section = Column("ui_section", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_key = Column("ui_key", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_value = Column("ui_value", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_active = Column("ui_active", Boolean(), nullable=True, unique=None, default=True)
 
 
 class User(Base, BaseModel):
     __tablename__ = 'users'
     __table_args__ = (UniqueConstraint('username'), UniqueConstraint('email'), {'useexisting':True})
     user_id = Column("user_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     username = Column("username", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     password = Column("password", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     active = Column("active", Boolean(), nullable=True, unique=None, default=None)
     admin = Column("admin", Boolean(), nullable=True, unique=None, default=False)
     name = Column("name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     lastname = Column("lastname", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     email = Column("email", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     last_login = Column("last_login", DateTime(timezone=False), nullable=True, unique=None, default=None)
     is_ldap = Column("is_ldap", Boolean(), nullable=False, unique=None, default=False)
 
     user_log = relation('UserLog', cascade='all')
     user_perms = relation('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id", cascade='all')
 
     repositories = relation('Repository')
     user_followers = relation('UserFollowing', primaryjoin='UserFollowing.follows_user_id==User.user_id', cascade='all')
 
     @property
     def full_contact(self):
         return '%s %s <%s>' % (self.name, self.lastname, self.email)
 
     def __repr__(self):
         return "<User('id:%s:%s')>" % (self.user_id, self.username)
 
     def update_lastlogin(self):
         """Update user lastlogin"""
 
         try:
             session = Session.object_session(self)
             self.last_login = datetime.datetime.now()
             session.add(self)
             session.commit()
             log.debug('updated user %s lastlogin', self.username)
         except (DatabaseError,):
             session.rollback()
 
 
 class UserLog(Base, BaseModel):
     __tablename__ = 'user_logs'
     __table_args__ = {'useexisting':True}
     user_log_id = Column("user_log_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
     repository_id = Column("repository_id", Integer(length=None, convert_unicode=False, assert_unicode=None), ForeignKey(u'repositories.repo_id'), nullable=False, unique=None, default=None)
     repository_name = Column("repository_name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     user_ip = Column("user_ip", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     action = Column("action", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     action_date = Column("action_date", DateTime(timezone=False), nullable=True, unique=None, default=None)
 
     user = relation('User')
     repository = relation('Repository')
 
 class Repository(Base, BaseModel):
     __tablename__ = 'repositories'
     __table_args__ = (UniqueConstraint('repo_name'), {'useexisting':True},)
     repo_id = Column("repo_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     repo_name = Column("repo_name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
-    repo_type = Column("repo_type", String(length=None, convert_unicode=False, assert_unicode=None), nullable=False, unique=False, default=None)
+    repo_type = Column("repo_type", String(length=None, convert_unicode=False, assert_unicode=None), nullable=False, unique=False, default='hg')
     user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=False, default=None)
     private = Column("private", Boolean(), nullable=True, unique=None, default=None)
     enable_statistics = Column("statistics", Boolean(), nullable=True, unique=None, default=True)
     description = Column("description", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     fork_id = Column("fork_id", Integer(), ForeignKey(u'repositories.repo_id'), nullable=True, unique=False, default=None)
 
     user = relation('User')
     fork = relation('Repository', remote_side=repo_id)
     repo_to_perm = relation('RepoToPerm', cascade='all')
     stats = relation('Statistics', cascade='all', uselist=False)
 
     repo_followers = relation('UserFollowing', primaryjoin='UserFollowing.follows_repo_id==Repository.repo_id', cascade='all')
 
 
     def __repr__(self):
         return "<Repository('%s:%s')>" % (self.repo_id, self.repo_name)
 
 class Permission(Base, BaseModel):
     __tablename__ = 'permissions'
     __table_args__ = {'useexisting':True}
     permission_id = Column("permission_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     permission_name = Column("permission_name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     permission_longname = Column("permission_longname", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
     def __repr__(self):
         return "<Permission('%s:%s')>" % (self.permission_id, self.permission_name)
 
 class RepoToPerm(Base, BaseModel):
     __tablename__ = 'repo_to_perm'
     __table_args__ = (UniqueConstraint('user_id', 'repository_id'), {'useexisting':True})
     repo_to_perm_id = Column("repo_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey(u'permissions.permission_id'), nullable=False, unique=None, default=None)
     repository_id = Column("repository_id", Integer(), ForeignKey(u'repositories.repo_id'), nullable=False, unique=None, default=None)
 
     user = relation('User')
     permission = relation('Permission')
     repository = relation('Repository')
 
 class UserToPerm(Base, BaseModel):
     __tablename__ = 'user_to_perm'
     __table_args__ = (UniqueConstraint('user_id', 'permission_id'), {'useexisting':True})
     user_to_perm_id = Column("user_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey(u'permissions.permission_id'), nullable=False, unique=None, default=None)
 
     user = relation('User')
     permission = relation('Permission')
 
 class Statistics(Base, BaseModel):
     __tablename__ = 'statistics'
     __table_args__ = (UniqueConstraint('repository_id'), {'useexisting':True})
     stat_id = Column("stat_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     repository_id = Column("repository_id", Integer(), ForeignKey(u'repositories.repo_id'), nullable=False, unique=True, default=None)
     stat_on_revision = Column("stat_on_revision", Integer(), nullable=False)
     commit_activity = Column("commit_activity", LargeBinary(), nullable=False)#JSON data
     commit_activity_combined = Column("commit_activity_combined", LargeBinary(), nullable=False)#JSON data
     languages = Column("languages", LargeBinary(), nullable=False)#JSON data
 
     repository = relation('Repository', single_parent=True)
 
 class UserFollowing(Base, BaseModel):
     __tablename__ = 'user_followings'
     __table_args__ = (UniqueConstraint('user_id', 'follows_repository_id'),
                       UniqueConstraint('user_id', 'follows_user_id')
                       , {'useexisting':True})
 
     user_following_id = Column("user_following_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
     follows_repo_id = Column("follows_repository_id", Integer(), ForeignKey(u'repositories.repo_id'), nullable=True, unique=None, default=None)
     follows_user_id = Column("follows_user_id", Integer(), ForeignKey(u'users.user_id'), nullable=True, unique=None, default=None)
 
     user = relation('User', primaryjoin='User.user_id==UserFollowing.user_id')
 
     follows_user = relation('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')
     follows_repository = relation('Repository')
 
 
 class CacheInvalidation(Base, BaseModel):
     __tablename__ = 'cache_invalidation'
     __table_args__ = (UniqueConstraint('cache_key'), {'useexisting':True})
     cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     cache_key = Column("cache_key", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     cache_args = Column("cache_args", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False)
 
 
     def __init__(self, cache_key, cache_args=''):
         self.cache_key = cache_key
         self.cache_args = cache_args
         self.cache_active = False
 
     def __repr__(self):
         return "<CacheInvalidation('%s:%s')>" % (self.cache_id, self.cache_key)
 
 class DbMigrateVersion(Base, BaseModel):
     __tablename__ = 'db_migrate_version'
     __table_args__ = {'useexisting':True}
     repository_id = Column('repository_id', String(250), primary_key=True)
     repository_path = Column('repository_path', Text)
     version = Column('version', Integer)