merged with beta branch

docs/api/index.rst

@@ -0,0 +1,10
+.. _api:
+
+API Reference
+=============
+
+.. toctree::
+   :maxdepth: 3
+
+   models
+    
  No newline at end of file

docs/api/models.rst

@@ -0,0 +1,19
+.. _models:
+
+The :mod:`models` Module
+========================
+
+.. automodule:: rhodecode.model
+   :members:
+   
+.. automodule:: rhodecode.model.permission
+   :members:
+  
+.. automodule:: rhodecode.model.repo
+   :members:   
+
+.. automodule:: rhodecode.model.scm
+   :members:
+
+.. automodule:: rhodecode.model.user
+   :members:      

docs/contributing.rst

@@ -0,0 +1,9
+.. _contributing:
+
+Contributing in RhodeCode
+=========================
+
+If You would like to contribute to RhodeCode, please contact me, any help is
+greatly appreciated.
+
+Thank You.

docs/enable_git.rst

@@ -0,0 +1,21
+.. _enable_git:
+
+Enabling GIT support (beta)
+===========================
+
+
+Git support in RhodeCode 1.1 was disabled due to some instability issues, but
+If You would like to test it fell free to re-enable it. To enable GIT just
+uncomment git line in rhodecode/__init__.py file
+
+.. code-block:: python
+ 
+   BACKENDS = {
+       'hg': 'Mercurial repository',
+       #'git': 'Git repository',
+   }
+
+.. note::
+   Please note that it's not fully stable and it might crash (that's why it 
+   was disabled), so be careful about enabling git support. Don't use it in 
+   production !
  No newline at end of file

docs/screenshots.rst

@@ -0,0 +1,13
+.. _screenshots:
+
+.. figure::  images/screenshot1_main_page.png
+
+   Main page of RhodeCode
+
+.. figure::  images/screenshot2_summary_page.png
+
+   Summary page
+   
+.. figure::  images/screenshot3_changelog_page.png
+
+   Changelog with DAG graph
  No newline at end of file

docs/statistics.rst

@@ -0,0 +1,32
+.. _statistics:
+
+
+Statistics
+==========
+
+RhodeCode statistics system is heavy on resources, so in order to keep a 
+balance between the usability and performance statistics are cached inside db
+and are gathered incrementally, this is how RhodeCode does this:
+
+With Celery disabled
+++++++++++++++++++++
+
+- on each first visit on summary page a set of 250 commits are parsed and
+  updates statistics cache
+- this happens on each single visit of statistics page until all commits are
+  fetched. Statistics are kept cached until some more commits are added to
+  repository, in such case RhodeCode will fetch only the ones added and will
+  update it's cache.
+
+
+With Celery enabled
++++++++++++++++++++
+
+- on first visit on summary page RhodeCode will create task that will execute
+  on celery workers, that will gather all stats until all commits are parsed,
+  each task will parse 250 commits, and run next task to parse next 250 
+  commits, until all are parsed.
+
+.. note::
+   In any time You can disable statistics on each repository in repository edit
+   form in admin panel, just uncheck the statistics checkbox.
  No newline at end of file

docs/theme/nature/layout.html

@@ -0,0 +1,14
+{% extends "basic/layout.html" %}
+
+{% block sidebarlogo %}
+<h3>Support my development effort.</h3>
+<div style="text-align:center">
+	<form action="https://www.paypal.com/cgi-bin/webscr" method="post">
+	<input type="hidden" name="cmd" value="_s-xclick">
+	<input type="hidden" name="hosted_button_id" value="8U2LLRPLBKWDU">
+	<input style="border:0px !important" type="image" src="https://www.paypal.com/en_US/i/btn/btn_donate_SM.gif" 
+	border="0" name="submit" alt="PayPal - The safer, easier way to pay online!">
+	<img alt="" border="0" src="https://www.paypal.com/en_US/i/scr/pixel.gif" width="1" height="1">
+	</form>
+</div>
+{% endblock %}}

rhodecode/controllers/admin/ldap_settings.py

@@ -0,0 +1,106
+# -*- coding: utf-8 -*-
+"""
+    package.rhodecode.controllers.admin.ldap_settings
+    ~~~~~~~~~~~~~~
+
+    ldap controller for RhodeCode
+    :created_on: Nov 26, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your opinion) any later version of the license.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+import logging
+import formencode
+import traceback
+
+from formencode import htmlfill
+
+from pylons import request, response, session, tmpl_context as c, url
+from pylons.controllers.util import abort, redirect
+from pylons.i18n.translation import _
+
+from rhodecode.lib.base import BaseController, render
+from rhodecode.lib import helpers as h
+from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator
+from rhodecode.lib.auth_ldap import LdapImportError
+from rhodecode.model.settings import SettingsModel
+from rhodecode.model.forms import LdapSettingsForm
+from sqlalchemy.exc import DatabaseError
+
+log = logging.getLogger(__name__)
+
+
+
+class LdapSettingsController(BaseController):
+
+    @LoginRequired()
+    @HasPermissionAllDecorator('hg.admin')
+    def __before__(self):
+        c.admin_user = session.get('admin_user')
+        c.admin_username = session.get('admin_username')
+        super(LdapSettingsController, self).__before__()
+
+    def index(self):
+        defaults = SettingsModel().get_ldap_settings()
+
+        return htmlfill.render(
+                    render('admin/ldap/ldap.html'),
+                    defaults=defaults,
+                    encoding="UTF-8",
+                    force_defaults=True,)
+
+    def ldap_settings(self):
+        """
+        POST ldap create and store ldap settings
+        """
+
+        settings_model = SettingsModel()
+        _form = LdapSettingsForm()()
+
+        try:
+            form_result = _form.to_python(dict(request.POST))
+            try:
+
+                for k, v in form_result.items():
+                    if k.startswith('ldap_'):
+                        setting = settings_model.get(k)
+                        setting.app_settings_value = v
+                        self.sa.add(setting)
+
+                self.sa.commit()
+                h.flash(_('Ldap settings updated successfully'),
+                    category='success')
+            except (DatabaseError,):
+                raise
+        except LdapImportError:
+            h.flash(_('Unable to activate ldap. The "python-ldap" library '
+                      'is missing.'), category='warning')
+
+        except formencode.Invalid, errors:
+
+            return htmlfill.render(
+                render('admin/ldap/ldap.html'),
+                defaults=errors.value,
+                errors=errors.error_dict or {},
+                prefix_error=False,
+                encoding="UTF-8")
+        except Exception:
+            log.error(traceback.format_exc())
+            h.flash(_('error occured during update of ldap settings'),
+                    category='error')
+
+        return redirect(url('ldap_home'))

rhodecode/controllers/journal.py

@@ -0,0 +1,93
+#!/usr/bin/env python
+# encoding: utf-8
+# journal controller for pylons
+# Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
+# 
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your opinion) any later version of the license.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+"""
+Created on November 21, 2010
+journal controller for pylons
+@author: marcink
+"""
+
+from pylons import request, response, session, tmpl_context as c, url
+from pylons.controllers.util import abort, redirect
+from rhodecode.lib.auth import LoginRequired, NotAnonymous
+from rhodecode.lib.base import BaseController, render
+from rhodecode.lib.helpers import get_token
+from rhodecode.model.db import UserLog, UserFollowing
+from rhodecode.model.scm import ScmModel
+from sqlalchemy import or_
+import logging
+from paste.httpexceptions import HTTPInternalServerError, HTTPNotFound
+
+log = logging.getLogger(__name__)
+
+class JournalController(BaseController):
+
+
+    @LoginRequired()
+    @NotAnonymous()
+    def __before__(self):
+        super(JournalController, self).__before__()
+
+    def index(self):
+        # Return a rendered template
+
+        c.following = self.sa.query(UserFollowing)\
+            .filter(UserFollowing.user_id == c.rhodecode_user.user_id).all()
+
+        repo_ids = [x.follows_repository.repo_id for x in c.following
+                    if x.follows_repository is not None]
+        user_ids = [x.follows_user.user_id for x in c.following
+                    if x.follows_user is not None]
+
+        c.journal = self.sa.query(UserLog)\
+            .filter(or_(
+                        UserLog.repository_id.in_(repo_ids),
+                        UserLog.user_id.in_(user_ids),
+                        ))\
+            .order_by(UserLog.action_date.desc())\
+            .limit(20)\
+            .all()
+        return render('/journal.html')
+
+    def toggle_following(self):
+
+        if request.POST.get('auth_token') == get_token():
+            scm_model = ScmModel()
+
+            user_id = request.POST.get('follows_user_id')
+            if user_id:
+                try:
+                    scm_model.toggle_following_user(user_id,
+                                                    c.rhodecode_user.user_id)
+                    return 'ok'
+                except:
+                    raise HTTPInternalServerError()
+
+            repo_id = request.POST.get('follows_repo_id')
+            if repo_id:
+                try:
+                    scm_model.toggle_following_repo(repo_id,
+                                                    c.rhodecode_user.user_id)
+                    return 'ok'
+                except:
+                    raise HTTPInternalServerError()
+
+
+
+        raise HTTPInternalServerError()

rhodecode/lib/auth_ldap.py

@@ -0,0 +1,104
+#!/usr/bin/env python
+# encoding: utf-8
+# ldap authentication lib
+# Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your opinion) any later version of the license.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+"""
+Created on Nov 17, 2010
+
+@author: marcink
+"""
+
+from rhodecode.lib.exceptions import *
+import logging
+
+log = logging.getLogger(__name__)
+
+try:
+    import ldap
+except ImportError:
+    pass
+
+class AuthLdap(object):
+
+    def __init__(self, server, base_dn, port=389, bind_dn='', bind_pass='',
+                 use_ldaps=False, ldap_version=3):
+        self.ldap_version = ldap_version
+        if use_ldaps:
+            port = port or 689
+        self.LDAP_USE_LDAPS = use_ldaps
+        self.LDAP_SERVER_ADDRESS = server
+        self.LDAP_SERVER_PORT = port
+
+        #USE FOR READ ONLY BIND TO LDAP SERVER
+        self.LDAP_BIND_DN = bind_dn
+        self.LDAP_BIND_PASS = bind_pass
+
+        ldap_server_type = 'ldap'
+        if self.LDAP_USE_LDAPS:ldap_server_type = ldap_server_type + 's'
+        self.LDAP_SERVER = "%s://%s:%s" % (ldap_server_type,
+                                               self.LDAP_SERVER_ADDRESS,
+                                               self.LDAP_SERVER_PORT)
+
+        self.BASE_DN = base_dn
+
+    def authenticate_ldap(self, username, password):
+        """Authenticate a user via LDAP and return his/her LDAP properties.
+    
+        Raises AuthenticationError if the credentials are rejected, or
+        EnvironmentError if the LDAP server can't be reached.
+        
+        :param username: username
+        :param password: password
+        """
+
+        from rhodecode.lib.helpers import chop_at
+
+        uid = chop_at(username, "@%s" % self.LDAP_SERVER_ADDRESS)
+
+        if "," in username:
+            raise LdapUsernameError("invalid character in username: ,")
+        try:
+            ldap.set_option(ldap.OPT_X_TLS_CACERTDIR, '/etc/openldap/cacerts')
+            ldap.set_option(ldap.OPT_NETWORK_TIMEOUT, 10)
+            server = ldap.initialize(self.LDAP_SERVER)
+            if self.ldap_version == 2:
+                server.protocol = ldap.VERSION2
+            else:
+                server.protocol = ldap.VERSION3
+
+            if self.LDAP_BIND_DN and self.LDAP_BIND_PASS:
+                server.simple_bind_s(self.LDAP_BIND_DN, self.LDAP_BIND_PASS)
+
+            dn = self.BASE_DN % {'user':uid}
+            log.debug("Authenticating %r at %s", dn, self.LDAP_SERVER)
+            server.simple_bind_s(dn, password)
+
+            properties = server.search_s(dn, ldap.SCOPE_SUBTREE)
+            if not properties:
+                raise ldap.NO_SUCH_OBJECT()
+        except ldap.NO_SUCH_OBJECT, e:
+            log.debug("LDAP says no such user '%s' (%s)", uid, username)
+            raise LdapUsernameError()
+        except ldap.INVALID_CREDENTIALS, e:
+            log.debug("LDAP rejected password for user '%s' (%s)", uid, username)
+            raise LdapPasswordError()
+        except ldap.SERVER_DOWN, e:
+            raise LdapConnectionError("LDAP can't access authentication server")
+
+        return properties[0]
+

rhodecode/lib/celerypylons/__init__.py

@@ -0,0 +1,16
+"""
+Automatically sets the environment variable `CELERY_LOADER` to
+`celerypylons.loader:PylonsLoader`.  This ensures the loader is
+specified when accessing the rest of this package, and allows celery
+to be installed in a webapp just by importing celerypylons::
+
+    import celerypylons
+
+"""
+import os
+import warnings
+
+CELERYPYLONS_LOADER = 'rhodecode.lib.celerypylons.loader.PylonsLoader'
+if os.environ.get('CELERY_LOADER', CELERYPYLONS_LOADER) != CELERYPYLONS_LOADER:
+    warnings.warn("'CELERY_LOADER' environment variable will be overridden by celery-pylons.")
+os.environ['CELERY_LOADER'] = CELERYPYLONS_LOADER

rhodecode/lib/celerypylons/commands.py

@@ -0,0 +1,90
+from rhodecode.lib.utils import BasePasterCommand, Command
+
+
+__all__ = ['CeleryDaemonCommand', 'CeleryBeatCommand',
+           'CAMQPAdminCommand', 'CeleryEventCommand']
+
+
+class CeleryDaemonCommand(BasePasterCommand):
+    """Start the celery worker
+
+    Starts the celery worker that uses a paste.deploy configuration
+    file.
+    """
+    usage = 'CONFIG_FILE [celeryd options...]'
+    summary = __doc__.splitlines()[0]
+    description = "".join(__doc__.splitlines()[2:])
+
+    parser = Command.standard_parser(quiet=True)
+
+    def update_parser(self):
+        from celery.bin import celeryd
+        for x in celeryd.WorkerCommand().get_options():
+            self.parser.add_option(x)
+
+    def command(self):
+        from celery.bin import celeryd
+        return celeryd.WorkerCommand().run(**vars(self.options))
+
+
+class CeleryBeatCommand(BasePasterCommand):
+    """Start the celery beat server
+
+    Starts the celery beat server using a paste.deploy configuration
+    file.
+    """
+    usage = 'CONFIG_FILE [celerybeat options...]'
+    summary = __doc__.splitlines()[0]
+    description = "".join(__doc__.splitlines()[2:])
+
+    parser = Command.standard_parser(quiet=True)
+
+    def update_parser(self):
+        from celery.bin import celerybeat
+        for x in celerybeat.BeatCommand().get_options():
+            self.parser.add_option(x)
+
+    def command(self):
+        from celery.bin import celerybeat
+        return celerybeat.BeatCommand(**vars(self.options))
+
+class CAMQPAdminCommand(BasePasterCommand):
+    """CAMQP Admin
+
+    CAMQP celery admin tool.
+    """
+    usage = 'CONFIG_FILE [camqadm options...]'
+    summary = __doc__.splitlines()[0]
+    description = "".join(__doc__.splitlines()[2:])
+
+    parser = Command.standard_parser(quiet=True)
+
+    def update_parser(self):
+        from celery.bin import camqadm
+        for x in camqadm.OPTION_LIST:
+            self.parser.add_option(x)
+
+    def command(self):
+        from celery.bin import camqadm
+        return camqadm.camqadm(*self.args, **vars(self.options))
+
+
+class CeleryEventCommand(BasePasterCommand):
+    """Celery event commandd.
+
+    Capture celery events.
+    """
+    usage = 'CONFIG_FILE [celeryev options...]'
+    summary = __doc__.splitlines()[0]
+    description = "".join(__doc__.splitlines()[2:])
+
+    parser = Command.standard_parser(quiet=True)
+
+    def update_parser(self):
+        from celery.bin import celeryev
+        for x in celeryev.OPTION_LIST:
+            self.parser.add_option(x)
+
+    def command(self):
+        from celery.bin import celeryev
+        return celeryev.run_celeryev(**vars(self.options))

rhodecode/lib/celerypylons/loader.py

@@ -0,0 +1,55
+from celery.loaders.base import BaseLoader
+from pylons import config
+
+to_pylons = lambda x: x.replace('_', '.').lower()
+to_celery = lambda x: x.replace('.', '_').upper()
+
+LIST_PARAMS = """CELERY_IMPORTS ADMINS ROUTES""".split()
+
+
+class PylonsSettingsProxy(object):
+    """Pylons Settings Proxy
+
+    Proxies settings from pylons.config
+
+    """
+    def __getattr__(self, key):
+        pylons_key = to_pylons(key)
+        try:
+            value = config[pylons_key]
+            if key in LIST_PARAMS: return value.split()
+            return self.type_converter(value)
+        except KeyError:
+            raise AttributeError(pylons_key)
+
+    def __setattr__(self, key, value):
+        pylons_key = to_pylons(key)
+        config[pylons_key] = value
+
+
+    def type_converter(self, value):
+        #cast to int
+        if value.isdigit():
+            return int(value)
+
+        #cast to bool
+        if value.lower() in ['true', 'false']:
+            return value.lower() == 'true'
+
+        return value
+
+class PylonsLoader(BaseLoader):
+    """Pylons celery loader
+
+    Maps the celery config onto pylons.config
+
+    """
+    def read_configuration(self):
+        self.configured = True
+        return PylonsSettingsProxy()
+
+    def on_worker_init(self):
+        """
+        Import task modules.
+        """
+        self.import_default_modules()

rhodecode/lib/dbmigrate/__init__.py

@@ -0,0 +1,69
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.lib.dbmigrate.__init__
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    
+    Database migration modules
+    
+    :created_on: Dec 11, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your opinion) any later version of the license.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+
+import logging
+from sqlalchemy import engine_from_config
+
+
+from rhodecode.lib.utils import BasePasterCommand, Command, add_cache
+from rhodecode.lib.db_manage import DbManage
+
+log = logging.getLogger(__name__)
+
+class UpgradeDb(BasePasterCommand):
+    """Command used for paster to upgrade our database to newer version
+    """
+
+    max_args = 1
+    min_args = 1
+
+    usage = "CONFIG_FILE"
+    summary = "Upgrades current db to newer version given configuration file"
+    group_name = "RhodeCode"
+
+    parser = Command.standard_parser(verbose=True)
+
+    def command(self):
+        from pylons import config
+
+        add_cache(config)
+
+        db_uri = config['sqlalchemy.db1.url']
+
+        dbmanage = DbManage(log_sql=True, dbconf=db_uri,
+                            root=config['here'], tests=False)
+
+        dbmanage.upgrade()
+
+
+
+    def update_parser(self):
+        self.parser.add_option('--sql',
+                      action='store_true',
+                      dest='just_sql',
+                      help="Prints upgrade sql for further investigation",
+                      default=False)

rhodecode/lib/dbmigrate/migrate.cfg

@@ -0,0 +1,20
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=rhodecode_db_migrations
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to 
+# change the table name in each database too. 
+version_table=db_migrate_version
+
+# When committing a change script, Migrate will attempt to generate the 
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the 
+# commit continues, perhaps ending successfully. 
+# Databases in this list MUST compile successfully during a commit, or the 
+# entire commit will fail. List the databases your application will actually 
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=['sqlite']

rhodecode/lib/dbmigrate/migrate/__init__.py

@@ -0,0 +1,9
+"""
+   SQLAlchemy migrate provides two APIs :mod:`migrate.versioning` for
+   database schema version and repository management and
+   :mod:`migrate.changeset` that allows to define database schema changes
+   using Python.
+"""
+
+from rhodecode.lib.dbmigrate.migrate.versioning import *
+from rhodecode.lib.dbmigrate.migrate.changeset import *

rhodecode/lib/dbmigrate/migrate/changeset/__init__.py

@@ -0,0 +1,28
+"""
+   This module extends SQLAlchemy and provides additional DDL [#]_
+   support.
+
+   .. [#] SQL Data Definition Language
+"""
+import re
+import warnings
+
+import sqlalchemy
+from sqlalchemy import __version__ as _sa_version
+
+warnings.simplefilter('always', DeprecationWarning)
+
+_sa_version = tuple(int(re.match("\d+", x).group(0)) for x in _sa_version.split("."))
+SQLA_06 = _sa_version >= (0, 6)
+
+del re
+del _sa_version
+
+from rhodecode.lib.dbmigrate.migrate.changeset.schema import *
+from rhodecode.lib.dbmigrate.migrate.changeset.constraint import *
+
+sqlalchemy.schema.Table.__bases__ += (ChangesetTable, )
+sqlalchemy.schema.Column.__bases__ += (ChangesetColumn, )
+sqlalchemy.schema.Index.__bases__ += (ChangesetIndex, )
+
+sqlalchemy.schema.DefaultClause.__bases__ += (ChangesetDefaultClause, )

rhodecode/lib/dbmigrate/migrate/changeset/ansisql.py

@@ -0,0 +1,358
+"""
+   Extensions to SQLAlchemy for altering existing tables.
+
+   At the moment, this isn't so much based off of ANSI as much as
+   things that just happen to work with multiple databases.
+"""
+import StringIO
+
+import sqlalchemy as sa
+from sqlalchemy.schema import SchemaVisitor
+from sqlalchemy.engine.default import DefaultDialect
+from sqlalchemy.sql import ClauseElement
+from sqlalchemy.schema import (ForeignKeyConstraint,
+                               PrimaryKeyConstraint,
+                               CheckConstraint,
+                               UniqueConstraint,
+                               Index)
+
+from rhodecode.lib.dbmigrate.migrate import exceptions
+from rhodecode.lib.dbmigrate.migrate.changeset import constraint, SQLA_06
+
+if not SQLA_06:
+    from sqlalchemy.sql.compiler import SchemaGenerator, SchemaDropper
+else:
+    from sqlalchemy.schema import AddConstraint, DropConstraint
+    from sqlalchemy.sql.compiler import DDLCompiler
+    SchemaGenerator = SchemaDropper = DDLCompiler
+
+
+class AlterTableVisitor(SchemaVisitor):
+    """Common operations for ``ALTER TABLE`` statements."""
+
+    if SQLA_06:
+        # engine.Compiler looks for .statement
+        # when it spawns off a new compiler
+        statement = ClauseElement()
+
+    def append(self, s):
+        """Append content to the SchemaIterator's query buffer."""
+
+        self.buffer.write(s)
+
+    def execute(self):
+        """Execute the contents of the SchemaIterator's buffer."""
+        try:
+            return self.connection.execute(self.buffer.getvalue())
+        finally:
+            self.buffer.truncate(0)
+
+    def __init__(self, dialect, connection, **kw):
+        self.connection = connection
+        self.buffer = StringIO.StringIO()
+        self.preparer = dialect.identifier_preparer
+        self.dialect = dialect
+
+    def traverse_single(self, elem):
+        ret = super(AlterTableVisitor, self).traverse_single(elem)
+        if ret:
+            # adapt to 0.6 which uses a string-returning
+            # object
+            self.append(" %s" % ret)
+            
+    def _to_table(self, param):
+        """Returns the table object for the given param object."""
+        if isinstance(param, (sa.Column, sa.Index, sa.schema.Constraint)):
+            ret = param.table
+        else:
+            ret = param
+        return ret
+
+    def start_alter_table(self, param):
+        """Returns the start of an ``ALTER TABLE`` SQL-Statement.
+
+        Use the param object to determine the table name and use it
+        for building the SQL statement.
+
+        :param param: object to determine the table from
+        :type param: :class:`sqlalchemy.Column`, :class:`sqlalchemy.Index`,
+          :class:`sqlalchemy.schema.Constraint`, :class:`sqlalchemy.Table`,
+          or string (table name)
+        """
+        table = self._to_table(param)
+        self.append('\nALTER TABLE %s ' % self.preparer.format_table(table))
+        return table
+
+
+class ANSIColumnGenerator(AlterTableVisitor, SchemaGenerator):
+    """Extends ansisql generator for column creation (alter table add col)"""
+
+    def visit_column(self, column):
+        """Create a column (table already exists).
+
+        :param column: column object
+        :type column: :class:`sqlalchemy.Column` instance
+        """
+        if column.default is not None:
+            self.traverse_single(column.default)
+
+        table = self.start_alter_table(column)
+        self.append("ADD ")
+        self.append(self.get_column_specification(column))
+
+        for cons in column.constraints:
+            self.traverse_single(cons)
+        self.execute()
+
+        # ALTER TABLE STATEMENTS
+
+        # add indexes and unique constraints
+        if column.index_name:
+            Index(column.index_name,column).create()
+        elif column.unique_name:
+            constraint.UniqueConstraint(column,
+                                        name=column.unique_name).create()
+
+        # SA bounds FK constraints to table, add manually
+        for fk in column.foreign_keys:
+            self.add_foreignkey(fk.constraint)
+
+        # add primary key constraint if needed
+        if column.primary_key_name:
+            cons = constraint.PrimaryKeyConstraint(column,
+                                                   name=column.primary_key_name)
+            cons.create()
+
+    if SQLA_06:
+        def add_foreignkey(self, fk):
+            self.connection.execute(AddConstraint(fk))
+
+class ANSIColumnDropper(AlterTableVisitor, SchemaDropper):
+    """Extends ANSI SQL dropper for column dropping (``ALTER TABLE
+    DROP COLUMN``).
+    """
+
+    def visit_column(self, column):
+        """Drop a column from its table.
+
+        :param column: the column object
+        :type column: :class:`sqlalchemy.Column`
+        """
+        table = self.start_alter_table(column)
+        self.append('DROP COLUMN %s' % self.preparer.format_column(column))
+        self.execute()
+
+
+class ANSISchemaChanger(AlterTableVisitor, SchemaGenerator):
+    """Manages changes to existing schema elements.
+
+    Note that columns are schema elements; ``ALTER TABLE ADD COLUMN``
+    is in SchemaGenerator.
+
+    All items may be renamed. Columns can also have many of their properties -
+    type, for example - changed.
+
+    Each function is passed a tuple, containing (object, name); where
+    object is a type of object you'd expect for that function
+    (ie. table for visit_table) and name is the object's new
+    name. NONE means the name is unchanged.
+    """
+
+    def visit_table(self, table):
+        """Rename a table. Other ops aren't supported."""
+        self.start_alter_table(table)
+        self.append("RENAME TO %s" % self.preparer.quote(table.new_name,
+                                                         table.quote))
+        self.execute()
+
+    def visit_index(self, index):
+        """Rename an index"""
+        if hasattr(self, '_validate_identifier'):
+            # SA <= 0.6.3
+            self.append("ALTER INDEX %s RENAME TO %s" % (
+                    self.preparer.quote(
+                        self._validate_identifier(
+                            index.name, True), index.quote),
+                    self.preparer.quote(
+                        self._validate_identifier(
+                            index.new_name, True), index.quote)))
+        else:
+            # SA >= 0.6.5
+            self.append("ALTER INDEX %s RENAME TO %s" % (
+                    self.preparer.quote(
+                        self._index_identifier(
+                            index.name), index.quote),
+                    self.preparer.quote(
+                        self._index_identifier(
+                            index.new_name), index.quote)))
+        self.execute()
+
+    def visit_column(self, delta):
+        """Rename/change a column."""
+        # ALTER COLUMN is implemented as several ALTER statements
+        keys = delta.keys()
+        if 'type' in keys:
+            self._run_subvisit(delta, self._visit_column_type)
+        if 'nullable' in keys:
+            self._run_subvisit(delta, self._visit_column_nullable)
+        if 'server_default' in keys:
+            # Skip 'default': only handle server-side defaults, others
+            # are managed by the app, not the db.
+            self._run_subvisit(delta, self._visit_column_default)
+        if 'name' in keys:
+            self._run_subvisit(delta, self._visit_column_name, start_alter=False)
+
+    def _run_subvisit(self, delta, func, start_alter=True):
+        """Runs visit method based on what needs to be changed on column"""
+        table = self._to_table(delta.table)
+        col_name = delta.current_name
+        if start_alter:
+            self.start_alter_column(table, col_name)
+        ret = func(table, delta.result_column, delta)
+        self.execute()
+
+    def start_alter_column(self, table, col_name):
+        """Starts ALTER COLUMN"""
+        self.start_alter_table(table)
+        self.append("ALTER COLUMN %s " % self.preparer.quote(col_name, table.quote))
+
+    def _visit_column_nullable(self, table, column, delta):
+        nullable = delta['nullable']
+        if nullable:
+            self.append("DROP NOT NULL")
+        else:
+            self.append("SET NOT NULL")
+
+    def _visit_column_default(self, table, column, delta):
+        default_text = self.get_column_default_string(column)
+        if default_text is not None:
+            self.append("SET DEFAULT %s" % default_text)
+        else:
+            self.append("DROP DEFAULT")
+
+    def _visit_column_type(self, table, column, delta):
+        type_ = delta['type']
+        if SQLA_06:
+            type_text = str(type_.compile(dialect=self.dialect))
+        else:
+            type_text = type_.dialect_impl(self.dialect).get_col_spec()
+        self.append("TYPE %s" % type_text)
+
+    def _visit_column_name(self, table, column, delta):
+        self.start_alter_table(table)
+        col_name = self.preparer.quote(delta.current_name, table.quote)
+        new_name = self.preparer.format_column(delta.result_column)
+        self.append('RENAME COLUMN %s TO %s' % (col_name, new_name))
+
+
+class ANSIConstraintCommon(AlterTableVisitor):
+    """
+    Migrate's constraints require a separate creation function from
+    SA's: Migrate's constraints are created independently of a table;
+    SA's are created at the same time as the table.
+    """
+
+    def get_constraint_name(self, cons):
+        """Gets a name for the given constraint.
+
+        If the name is already set it will be used otherwise the
+        constraint's :meth:`autoname <migrate.changeset.constraint.ConstraintChangeset.autoname>`
+        method is used.
+
+        :param cons: constraint object
+        """
+        if cons.name is not None:
+            ret = cons.name
+        else:
+            ret = cons.name = cons.autoname()
+        return self.preparer.quote(ret, cons.quote)
+
+    def visit_migrate_primary_key_constraint(self, *p, **k):
+        self._visit_constraint(*p, **k)
+
+    def visit_migrate_foreign_key_constraint(self, *p, **k):
+        self._visit_constraint(*p, **k)
+
+    def visit_migrate_check_constraint(self, *p, **k):
+        self._visit_constraint(*p, **k)
+
+    def visit_migrate_unique_constraint(self, *p, **k):
+        self._visit_constraint(*p, **k)
+
+if SQLA_06:
+    class ANSIConstraintGenerator(ANSIConstraintCommon, SchemaGenerator):
+        def _visit_constraint(self, constraint):
+            constraint.name = self.get_constraint_name(constraint)
+            self.append(self.process(AddConstraint(constraint)))
+            self.execute()
+
+    class ANSIConstraintDropper(ANSIConstraintCommon, SchemaDropper):
+        def _visit_constraint(self, constraint):
+            constraint.name = self.get_constraint_name(constraint)
+            self.append(self.process(DropConstraint(constraint, cascade=constraint.cascade)))
+            self.execute()
+
+else:
+    class ANSIConstraintGenerator(ANSIConstraintCommon, SchemaGenerator):
+
+        def get_constraint_specification(self, cons, **kwargs):
+            """Constaint SQL generators.
+        
+            We cannot use SA visitors because they append comma.
+            """
+        
+            if isinstance(cons, PrimaryKeyConstraint):
+                if cons.name is not None:
+                    self.append("CONSTRAINT %s " % self.preparer.format_constraint(cons))
+                self.append("PRIMARY KEY ")
+                self.append("(%s)" % ', '.join(self.preparer.quote(c.name, c.quote)
+                                               for c in cons))
+                self.define_constraint_deferrability(cons)
+            elif isinstance(cons, ForeignKeyConstraint):
+                self.define_foreign_key(cons)
+            elif isinstance(cons, CheckConstraint):
+                if cons.name is not None:
+                    self.append("CONSTRAINT %s " %
+                                self.preparer.format_constraint(cons))
+                self.append("CHECK (%s)" % cons.sqltext)
+                self.define_constraint_deferrability(cons)
+            elif isinstance(cons, UniqueConstraint):
+                if cons.name is not None:
+                    self.append("CONSTRAINT %s " %
+                                self.preparer.format_constraint(cons))
+                self.append("UNIQUE (%s)" % \
+                    (', '.join(self.preparer.quote(c.name, c.quote) for c in cons)))
+                self.define_constraint_deferrability(cons)
+            else:
+                raise exceptions.InvalidConstraintError(cons)
+
+        def _visit_constraint(self, constraint):
+        
+            table = self.start_alter_table(constraint)
+            constraint.name = self.get_constraint_name(constraint)
+            self.append("ADD ")
+            self.get_constraint_specification(constraint)
+            self.execute()
+    
+
+    class ANSIConstraintDropper(ANSIConstraintCommon, SchemaDropper):
+
+        def _visit_constraint(self, constraint):
+            self.start_alter_table(constraint)
+            self.append("DROP CONSTRAINT ")
+            constraint.name = self.get_constraint_name(constraint)
+            self.append(self.preparer.format_constraint(constraint))
+            if constraint.cascade:
+                self.cascade_constraint(constraint)
+            self.execute()
+
+        def cascade_constraint(self, constraint):
+            self.append(" CASCADE")
+
+
+class ANSIDialect(DefaultDialect):
+    columngenerator = ANSIColumnGenerator
+    columndropper = ANSIColumnDropper
+    schemachanger = ANSISchemaChanger
+    constraintgenerator = ANSIConstraintGenerator
+    constraintdropper = ANSIConstraintDropper

rhodecode/lib/dbmigrate/migrate/changeset/constraint.py

@@ -0,0 +1,202
+"""
+   This module defines standalone schema constraint classes.
+"""
+from sqlalchemy import schema
+
+from rhodecode.lib.dbmigrate.migrate.exceptions import *
+from rhodecode.lib.dbmigrate.migrate.changeset import SQLA_06
+
+class ConstraintChangeset(object):
+    """Base class for Constraint classes."""
+
+    def _normalize_columns(self, cols, table_name=False):
+        """Given: column objects or names; return col names and
+        (maybe) a table"""
+        colnames = []
+        table = None
+        for col in cols:
+            if isinstance(col, schema.Column):
+                if col.table is not None and table is None:
+                    table = col.table
+                if table_name:
+                    col = '.'.join((col.table.name, col.name))
+                else:
+                    col = col.name
+            colnames.append(col)
+        return colnames, table
+
+    def __do_imports(self, visitor_name, *a, **kw):
+        engine = kw.pop('engine', self.table.bind)
+        from rhodecode.lib.dbmigrate.migrate.changeset.databases.visitor import (get_engine_visitor,
+                                                         run_single_visitor)
+        visitorcallable = get_engine_visitor(engine, visitor_name)
+        run_single_visitor(engine, visitorcallable, self, *a, **kw)
+
+    def create(self, *a, **kw):
+        """Create the constraint in the database.
+
+        :param engine: the database engine to use. If this is \
+        :keyword:`None` the instance's engine will be used
+        :type engine: :class:`sqlalchemy.engine.base.Engine`
+        :param connection: reuse connection istead of creating new one.
+        :type connection: :class:`sqlalchemy.engine.base.Connection` instance
+        """
+        # TODO: set the parent here instead of in __init__
+        self.__do_imports('constraintgenerator', *a, **kw)
+
+    def drop(self, *a, **kw):
+        """Drop the constraint from the database.
+
+        :param engine: the database engine to use. If this is
+          :keyword:`None` the instance's engine will be used
+        :param cascade: Issue CASCADE drop if database supports it
+        :type engine: :class:`sqlalchemy.engine.base.Engine`
+        :type cascade: bool
+        :param connection: reuse connection istead of creating new one.
+        :type connection: :class:`sqlalchemy.engine.base.Connection` instance
+        :returns: Instance with cleared columns
+        """
+        self.cascade = kw.pop('cascade', False)
+        self.__do_imports('constraintdropper', *a, **kw)
+        # the spirit of Constraint objects is that they
+        # are immutable (just like in a DB.  they're only ADDed
+        # or DROPped).
+        #self.columns.clear()
+        return self
+
+
+class PrimaryKeyConstraint(ConstraintChangeset, schema.PrimaryKeyConstraint):
+    """Construct PrimaryKeyConstraint
+
+    Migrate's additional parameters:
+
+    :param cols: Columns in constraint.
+    :param table: If columns are passed as strings, this kw is required
+    :type table: Table instance
+    :type cols: strings or Column instances
+    """
+
+    __migrate_visit_name__ = 'migrate_primary_key_constraint'
+
+    def __init__(self, *cols, **kwargs):
+        colnames, table = self._normalize_columns(cols)
+        table = kwargs.pop('table', table)
+        super(PrimaryKeyConstraint, self).__init__(*colnames, **kwargs)
+        if table is not None:
+            self._set_parent(table)
+
+
+    def autoname(self):
+        """Mimic the database's automatic constraint names"""
+        return "%s_pkey" % self.table.name
+
+
+class ForeignKeyConstraint(ConstraintChangeset, schema.ForeignKeyConstraint):
+    """Construct ForeignKeyConstraint
+
+    Migrate's additional parameters:
+
+    :param columns: Columns in constraint
+    :param refcolumns: Columns that this FK reffers to in another table.
+    :param table: If columns are passed as strings, this kw is required
+    :type table: Table instance
+    :type columns: list of strings or Column instances
+    :type refcolumns: list of strings or Column instances
+    """
+
+    __migrate_visit_name__ = 'migrate_foreign_key_constraint'
+
+    def __init__(self, columns, refcolumns, *args, **kwargs):
+        colnames, table = self._normalize_columns(columns)
+        table = kwargs.pop('table', table)
+        refcolnames, reftable = self._normalize_columns(refcolumns,
+                                                        table_name=True)
+        super(ForeignKeyConstraint, self).__init__(colnames, refcolnames, *args,
+                                                   **kwargs)
+        if table is not None:
+            self._set_parent(table)
+
+    @property
+    def referenced(self):
+        return [e.column for e in self.elements]
+
+    @property
+    def reftable(self):
+        return self.referenced[0].table
+
+    def autoname(self):
+        """Mimic the database's automatic constraint names"""
+        if hasattr(self.columns, 'keys'):
+            # SA <= 0.5
+            firstcol = self.columns[self.columns.keys()[0]]
+            ret = "%(table)s_%(firstcolumn)s_fkey" % dict(
+                table=firstcol.table.name,
+                firstcolumn=firstcol.name,)
+        else:
+            # SA >= 0.6
+            ret = "%(table)s_%(firstcolumn)s_fkey" % dict(
+                table=self.table.name,
+                firstcolumn=self.columns[0],)
+        return ret
+
+
+class CheckConstraint(ConstraintChangeset, schema.CheckConstraint):
+    """Construct CheckConstraint
+
+    Migrate's additional parameters:
+
+    :param sqltext: Plain SQL text to check condition
+    :param columns: If not name is applied, you must supply this kw\
+    to autoname constraint
+    :param table: If columns are passed as strings, this kw is required
+    :type table: Table instance
+    :type columns: list of Columns instances
+    :type sqltext: string
+    """
+
+    __migrate_visit_name__ = 'migrate_check_constraint'
+
+    def __init__(self, sqltext, *args, **kwargs):
+        cols = kwargs.pop('columns', [])
+        if not cols and not kwargs.get('name', False):
+            raise InvalidConstraintError('You must either set "name"'
+                'parameter or "columns" to autogenarate it.')
+        colnames, table = self._normalize_columns(cols)
+        table = kwargs.pop('table', table)
+        schema.CheckConstraint.__init__(self, sqltext, *args, **kwargs)
+        if table is not None:
+            if not SQLA_06:
+                self.table = table
+            self._set_parent(table)
+        self.colnames = colnames
+
+    def autoname(self):
+        return "%(table)s_%(cols)s_check" % \
+            dict(table=self.table.name, cols="_".join(self.colnames))
+
+
+class UniqueConstraint(ConstraintChangeset, schema.UniqueConstraint):
+    """Construct UniqueConstraint
+
+    Migrate's additional parameters:
+
+    :param cols: Columns in constraint.
+    :param table: If columns are passed as strings, this kw is required
+    :type table: Table instance
+    :type cols: strings or Column instances
+
+    .. versionadded:: 0.6.0
+    """
+
+    __migrate_visit_name__ = 'migrate_unique_constraint'
+
+    def __init__(self, *cols, **kwargs):
+        self.colnames, table = self._normalize_columns(cols)
+        table = kwargs.pop('table', table)
+        super(UniqueConstraint, self).__init__(*self.colnames, **kwargs)
+        if table is not None:
+            self._set_parent(table)
+
+    def autoname(self):
+        """Mimic the database's automatic constraint names"""
+        return "%s_%s_key" % (self.table.name, self.colnames[0])

rhodecode/lib/dbmigrate/migrate/changeset/databases/__init__.py

@@ -0,0 +1,10
+"""
+   This module contains database dialect specific changeset
+   implementations.
+"""
+__all__ = [
+    'postgres',
+    'sqlite',
+    'mysql',
+    'oracle',
+]

rhodecode/lib/dbmigrate/migrate/changeset/databases/firebird.py

@@ -0,0 +1,80
+"""
+   Firebird database specific implementations of changeset classes.
+"""
+from sqlalchemy.databases import firebird as sa_base
+
+from rhodecode.lib.dbmigrate.migrate import exceptions
+from rhodecode.lib.dbmigrate.migrate.changeset import ansisql, SQLA_06
+
+
+if SQLA_06:
+    FBSchemaGenerator = sa_base.FBDDLCompiler
+else:
+    FBSchemaGenerator = sa_base.FBSchemaGenerator
+
+class FBColumnGenerator(FBSchemaGenerator, ansisql.ANSIColumnGenerator):
+    """Firebird column generator implementation."""
+
+
+class FBColumnDropper(ansisql.ANSIColumnDropper):
+    """Firebird column dropper implementation."""
+
+    def visit_column(self, column):
+        """Firebird supports 'DROP col' instead of 'DROP COLUMN col' syntax
+
+        Drop primary key and unique constraints if dropped column is referencing it."""
+        if column.primary_key:
+            if column.table.primary_key.columns.contains_column(column):
+                column.table.primary_key.drop()
+                # TODO: recreate primary key if it references more than this column
+        if column.unique or getattr(column, 'unique_name', None):
+            for cons in column.table.constraints:
+                if cons.contains_column(column):
+                    cons.drop()
+                    # TODO: recreate unique constraint if it refenrences more than this column
+
+        table = self.start_alter_table(column)
+        self.append('DROP %s' % self.preparer.format_column(column))
+        self.execute()
+
+
+class FBSchemaChanger(ansisql.ANSISchemaChanger):
+    """Firebird schema changer implementation."""
+
+    def visit_table(self, table):
+        """Rename table not supported"""
+        raise exceptions.NotSupportedError(
+            "Firebird does not support renaming tables.")
+
+    def _visit_column_name(self, table, column, delta):
+        self.start_alter_table(table)
+        col_name = self.preparer.quote(delta.current_name, table.quote)
+        new_name = self.preparer.format_column(delta.result_column)
+        self.append('ALTER COLUMN %s TO %s' % (col_name, new_name))
+
+    def _visit_column_nullable(self, table, column, delta):
+        """Changing NULL is not supported"""
+        # TODO: http://www.firebirdfaq.org/faq103/
+        raise exceptions.NotSupportedError(
+            "Firebird does not support altering NULL bevahior.")
+
+
+class FBConstraintGenerator(ansisql.ANSIConstraintGenerator):
+    """Firebird constraint generator implementation."""
+
+
+class FBConstraintDropper(ansisql.ANSIConstraintDropper):
+    """Firebird constaint dropper implementation."""
+
+    def cascade_constraint(self, constraint):
+        """Cascading constraints is not supported"""
+        raise exceptions.NotSupportedError(
+            "Firebird does not support cascading constraints")
+
+
+class FBDialect(ansisql.ANSIDialect):
+    columngenerator = FBColumnGenerator
+    columndropper = FBColumnDropper
+    schemachanger = FBSchemaChanger
+    constraintgenerator = FBConstraintGenerator
+    constraintdropper = FBConstraintDropper

rhodecode/lib/dbmigrate/migrate/changeset/databases/mysql.py

@@ -0,0 +1,94
+"""
+   MySQL database specific implementations of changeset classes.
+"""
+
+from sqlalchemy.databases import mysql as sa_base
+from sqlalchemy import types as sqltypes
+
+from rhodecode.lib.dbmigrate.migrate import exceptions
+from rhodecode.lib.dbmigrate.migrate.changeset import ansisql, SQLA_06
+
+
+if not SQLA_06:
+    MySQLSchemaGenerator = sa_base.MySQLSchemaGenerator
+else:
+    MySQLSchemaGenerator = sa_base.MySQLDDLCompiler
+
+class MySQLColumnGenerator(MySQLSchemaGenerator, ansisql.ANSIColumnGenerator):
+    pass
+
+
+class MySQLColumnDropper(ansisql.ANSIColumnDropper):
+    pass
+
+
+class MySQLSchemaChanger(MySQLSchemaGenerator, ansisql.ANSISchemaChanger):
+
+    def visit_column(self, delta):
+        table = delta.table
+        colspec = self.get_column_specification(delta.result_column)
+        if delta.result_column.autoincrement:
+            primary_keys = [c for c in table.primary_key.columns
+                       if (c.autoincrement and
+                            isinstance(c.type, sqltypes.Integer) and
+                            not c.foreign_keys)]
+
+            if primary_keys:
+                first = primary_keys.pop(0)
+                if first.name == delta.current_name:
+                    colspec += " AUTO_INCREMENT"
+        old_col_name = self.preparer.quote(delta.current_name, table.quote)
+
+        self.start_alter_table(table)
+
+        self.append("CHANGE COLUMN %s " % old_col_name)
+        self.append(colspec)
+        self.execute()
+
+    def visit_index(self, param):
+        # If MySQL can do this, I can't find how
+        raise exceptions.NotSupportedError("MySQL cannot rename indexes")
+
+
+class MySQLConstraintGenerator(ansisql.ANSIConstraintGenerator):
+    pass
+
+if SQLA_06:
+    class MySQLConstraintDropper(MySQLSchemaGenerator, ansisql.ANSIConstraintDropper):
+        def visit_migrate_check_constraint(self, *p, **k):
+            raise exceptions.NotSupportedError("MySQL does not support CHECK"
+                " constraints, use triggers instead.")
+
+else:
+    class MySQLConstraintDropper(ansisql.ANSIConstraintDropper):
+
+        def visit_migrate_primary_key_constraint(self, constraint):
+            self.start_alter_table(constraint)
+            self.append("DROP PRIMARY KEY")
+            self.execute()
+
+        def visit_migrate_foreign_key_constraint(self, constraint):
+            self.start_alter_table(constraint)
+            self.append("DROP FOREIGN KEY ")
+            constraint.name = self.get_constraint_name(constraint)
+            self.append(self.preparer.format_constraint(constraint))
+            self.execute()
+
+        def visit_migrate_check_constraint(self, *p, **k):
+            raise exceptions.NotSupportedError("MySQL does not support CHECK"
+                " constraints, use triggers instead.")
+
+        def visit_migrate_unique_constraint(self, constraint, *p, **k):
+            self.start_alter_table(constraint)
+            self.append('DROP INDEX ')
+            constraint.name = self.get_constraint_name(constraint)
+            self.append(self.preparer.format_constraint(constraint))
+            self.execute()
+
+
+class MySQLDialect(ansisql.ANSIDialect):
+    columngenerator = MySQLColumnGenerator
+    columndropper = MySQLColumnDropper
+    schemachanger = MySQLSchemaChanger
+    constraintgenerator = MySQLConstraintGenerator
+    constraintdropper = MySQLConstraintDropper

rhodecode/lib/dbmigrate/migrate/changeset/databases/oracle.py

@@ -0,0 +1,111
+"""
+   Oracle database specific implementations of changeset classes.
+"""
+import sqlalchemy as sa
+from sqlalchemy.databases import oracle as sa_base
+
+from rhodecode.lib.dbmigrate.migrate import exceptions
+from rhodecode.lib.dbmigrate.migrate.changeset import ansisql, SQLA_06
+
+
+if not SQLA_06:
+    OracleSchemaGenerator = sa_base.OracleSchemaGenerator
+else:
+    OracleSchemaGenerator = sa_base.OracleDDLCompiler
+
+
+class OracleColumnGenerator(OracleSchemaGenerator, ansisql.ANSIColumnGenerator):
+    pass
+
+
+class OracleColumnDropper(ansisql.ANSIColumnDropper):
+    pass
+
+
+class OracleSchemaChanger(OracleSchemaGenerator, ansisql.ANSISchemaChanger):
+
+    def get_column_specification(self, column, **kwargs):
+        # Ignore the NOT NULL generated
+        override_nullable = kwargs.pop('override_nullable', None)
+        if override_nullable:
+            orig = column.nullable
+            column.nullable = True
+        ret = super(OracleSchemaChanger, self).get_column_specification(
+            column, **kwargs)
+        if override_nullable:
+            column.nullable = orig
+        return ret
+
+    def visit_column(self, delta):
+        keys = delta.keys()
+
+        if 'name' in keys:
+            self._run_subvisit(delta,
+                               self._visit_column_name,
+                               start_alter=False)
+
+        if len(set(('type', 'nullable', 'server_default')).intersection(keys)):
+            self._run_subvisit(delta,
+                               self._visit_column_change,
+                               start_alter=False)
+
+    def _visit_column_change(self, table, column, delta):
+        # Oracle cannot drop a default once created, but it can set it
+        # to null.  We'll do that if default=None
+        # http://forums.oracle.com/forums/message.jspa?messageID=1273234#1273234
+        dropdefault_hack = (column.server_default is None \
+                                and 'server_default' in delta.keys())
+        # Oracle apparently doesn't like it when we say "not null" if
+        # the column's already not null. Fudge it, so we don't need a
+        # new function
+        notnull_hack = ((not column.nullable) \
+                            and ('nullable' not in delta.keys()))
+        # We need to specify NULL if we're removing a NOT NULL
+        # constraint
+        null_hack = (column.nullable and ('nullable' in delta.keys()))
+
+        if dropdefault_hack:
+            column.server_default = sa.PassiveDefault(sa.sql.null())
+        if notnull_hack:
+            column.nullable = True
+        colspec = self.get_column_specification(column,
+            override_nullable=null_hack)
+        if null_hack:
+            colspec += ' NULL'
+        if notnull_hack:
+            column.nullable = False
+        if dropdefault_hack:
+            column.server_default = None
+
+        self.start_alter_table(table)
+        self.append("MODIFY (")
+        self.append(colspec)
+        self.append(")")
+
+
+class OracleConstraintCommon(object):
+
+    def get_constraint_name(self, cons):
+        # Oracle constraints can't guess their name like other DBs
+        if not cons.name:
+            raise exceptions.NotSupportedError(
+                "Oracle constraint names must be explicitly stated")
+        return cons.name
+
+
+class OracleConstraintGenerator(OracleConstraintCommon,
+                                ansisql.ANSIConstraintGenerator):
+    pass
+
+
+class OracleConstraintDropper(OracleConstraintCommon,
+                              ansisql.ANSIConstraintDropper):
+    pass
+
+
+class OracleDialect(ansisql.ANSIDialect):
+    columngenerator = OracleColumnGenerator
+    columndropper = OracleColumnDropper
+    schemachanger = OracleSchemaChanger
+    constraintgenerator = OracleConstraintGenerator
+    constraintdropper = OracleConstraintDropper

rhodecode/lib/dbmigrate/migrate/changeset/databases/postgres.py

@@ -0,0 +1,46
+"""
+   `PostgreSQL`_ database specific implementations of changeset classes.
+
+   .. _`PostgreSQL`: http://www.postgresql.org/
+"""
+from rhodecode.lib.dbmigrate.migrate.changeset import ansisql, SQLA_06
+
+if not SQLA_06:
+    from sqlalchemy.databases import postgres as sa_base
+    PGSchemaGenerator = sa_base.PGSchemaGenerator
+else:
+    from sqlalchemy.databases import postgresql as sa_base
+    PGSchemaGenerator = sa_base.PGDDLCompiler
+
+
+class PGColumnGenerator(PGSchemaGenerator, ansisql.ANSIColumnGenerator):
+    """PostgreSQL column generator implementation."""
+    pass
+
+
+class PGColumnDropper(ansisql.ANSIColumnDropper):
+    """PostgreSQL column dropper implementation."""
+    pass
+
+
+class PGSchemaChanger(ansisql.ANSISchemaChanger):
+    """PostgreSQL schema changer implementation."""
+    pass
+
+
+class PGConstraintGenerator(ansisql.ANSIConstraintGenerator):
+    """PostgreSQL constraint generator implementation."""
+    pass
+
+
+class PGConstraintDropper(ansisql.ANSIConstraintDropper):
+    """PostgreSQL constaint dropper implementation."""
+    pass
+
+
+class PGDialect(ansisql.ANSIDialect):
+    columngenerator = PGColumnGenerator
+    columndropper = PGColumnDropper
+    schemachanger = PGSchemaChanger
+    constraintgenerator = PGConstraintGenerator
+    constraintdropper = PGConstraintDropper

rhodecode/lib/dbmigrate/migrate/changeset/databases/sqlite.py

@@ -0,0 +1,148
+"""
+   `SQLite`_ database specific implementations of changeset classes.
+
+   .. _`SQLite`: http://www.sqlite.org/
+"""
+from UserDict import DictMixin
+from copy import copy
+
+from sqlalchemy.databases import sqlite as sa_base
+
+from rhodecode.lib.dbmigrate.migrate import exceptions
+from rhodecode.lib.dbmigrate.migrate.changeset import ansisql, SQLA_06
+
+
+if not SQLA_06:
+    SQLiteSchemaGenerator = sa_base.SQLiteSchemaGenerator
+else:
+    SQLiteSchemaGenerator = sa_base.SQLiteDDLCompiler
+
+class SQLiteCommon(object):
+
+    def _not_supported(self, op):
+        raise exceptions.NotSupportedError("SQLite does not support "
+            "%s; see http://www.sqlite.org/lang_altertable.html" % op)
+
+
+class SQLiteHelper(SQLiteCommon):
+
+    def recreate_table(self,table,column=None,delta=None):
+        table_name = self.preparer.format_table(table)
+
+        # we remove all indexes so as not to have
+        # problems during copy and re-create
+        for index in table.indexes:
+            index.drop()
+
+        self.append('ALTER TABLE %s RENAME TO migration_tmp' % table_name)
+        self.execute()
+
+        insertion_string = self._modify_table(table, column, delta)
+
+        table.create()
+        self.append(insertion_string % {'table_name': table_name})
+        self.execute()
+        self.append('DROP TABLE migration_tmp')
+        self.execute()
+        
+    def visit_column(self, delta):
+        if isinstance(delta, DictMixin):
+            column = delta.result_column
+            table = self._to_table(delta.table)
+        else:
+            column = delta
+            table = self._to_table(column.table)
+        self.recreate_table(table,column,delta)
+
+class SQLiteColumnGenerator(SQLiteSchemaGenerator, 
+                            ansisql.ANSIColumnGenerator,
+                            # at the end so we get the normal
+                            # visit_column by default
+                            SQLiteHelper,
+                            SQLiteCommon
+                            ):
+    """SQLite ColumnGenerator"""
+
+    def _modify_table(self, table, column, delta):
+        columns = ' ,'.join(map(
+                self.preparer.format_column,
+                [c for c in table.columns if c.name!=column.name]))
+        return ('INSERT INTO %%(table_name)s (%(cols)s) '
+                'SELECT %(cols)s from migration_tmp')%{'cols':columns}
+
+    def visit_column(self,column):
+        if column.foreign_keys:
+            SQLiteHelper.visit_column(self,column)
+        else:
+            super(SQLiteColumnGenerator,self).visit_column(column)
+
+class SQLiteColumnDropper(SQLiteHelper, ansisql.ANSIColumnDropper):
+    """SQLite ColumnDropper"""
+
+    def _modify_table(self, table, column, delta):
+        columns = ' ,'.join(map(self.preparer.format_column, table.columns))
+        return 'INSERT INTO %(table_name)s SELECT ' + columns + \
+            ' from migration_tmp'
+
+
+class SQLiteSchemaChanger(SQLiteHelper, ansisql.ANSISchemaChanger):
+    """SQLite SchemaChanger"""
+
+    def _modify_table(self, table, column, delta):
+        return 'INSERT INTO %(table_name)s SELECT * from migration_tmp'
+
+    def visit_index(self, index):
+        """Does not support ALTER INDEX"""
+        self._not_supported('ALTER INDEX')
+
+
+class SQLiteConstraintGenerator(ansisql.ANSIConstraintGenerator, SQLiteHelper, SQLiteCommon):
+
+    def visit_migrate_primary_key_constraint(self, constraint):
+        tmpl = "CREATE UNIQUE INDEX %s ON %s ( %s )"
+        cols = ', '.join(map(self.preparer.format_column, constraint.columns))
+        tname = self.preparer.format_table(constraint.table)
+        name = self.get_constraint_name(constraint)
+        msg = tmpl % (name, tname, cols)
+        self.append(msg)
+        self.execute()
+
+    def _modify_table(self, table, column, delta):
+        return 'INSERT INTO %(table_name)s SELECT * from migration_tmp'
+
+    def visit_migrate_foreign_key_constraint(self, *p, **k):
+        self.recreate_table(p[0].table)
+
+    def visit_migrate_unique_constraint(self, *p, **k):
+        self.recreate_table(p[0].table)
+
+
+class SQLiteConstraintDropper(ansisql.ANSIColumnDropper,
+                              SQLiteCommon,
+                              ansisql.ANSIConstraintCommon):
+
+    def visit_migrate_primary_key_constraint(self, constraint):
+        tmpl = "DROP INDEX %s "
+        name = self.get_constraint_name(constraint)
+        msg = tmpl % (name)
+        self.append(msg)
+        self.execute()
+
+    def visit_migrate_foreign_key_constraint(self, *p, **k):
+        self._not_supported('ALTER TABLE DROP CONSTRAINT')
+
+    def visit_migrate_check_constraint(self, *p, **k):
+        self._not_supported('ALTER TABLE DROP CONSTRAINT')
+
+    def visit_migrate_unique_constraint(self, *p, **k):
+        self._not_supported('ALTER TABLE DROP CONSTRAINT')
+
+
+# TODO: technically primary key is a NOT NULL + UNIQUE constraint, should add NOT NULL to index
+
+class SQLiteDialect(ansisql.ANSIDialect):
+    columngenerator = SQLiteColumnGenerator
+    columndropper = SQLiteColumnDropper
+    schemachanger = SQLiteSchemaChanger
+    constraintgenerator = SQLiteConstraintGenerator
+    constraintdropper = SQLiteConstraintDropper

rhodecode/lib/dbmigrate/migrate/changeset/databases/visitor.py

@@ -0,0 +1,78
+"""
+   Module for visitor class mapping.
+"""
+import sqlalchemy as sa
+
+from rhodecode.lib.dbmigrate.migrate.changeset import ansisql
+from rhodecode.lib.dbmigrate.migrate.changeset.databases import (sqlite,
+                                         postgres,
+                                         mysql,
+                                         oracle,
+                                         firebird)
+
+
+# Map SA dialects to the corresponding Migrate extensions
+DIALECTS = {
+    "default": ansisql.ANSIDialect,
+    "sqlite": sqlite.SQLiteDialect,
+    "postgres": postgres.PGDialect,
+    "postgresql": postgres.PGDialect,
+    "mysql": mysql.MySQLDialect,
+    "oracle": oracle.OracleDialect,
+    "firebird": firebird.FBDialect,
+}
+
+
+def get_engine_visitor(engine, name):
+    """
+    Get the visitor implementation for the given database engine.
+
+    :param engine: SQLAlchemy Engine
+    :param name: Name of the visitor
+    :type name: string
+    :type engine: Engine
+    :returns: visitor
+    """
+    # TODO: link to supported visitors
+    return get_dialect_visitor(engine.dialect, name)
+
+
+def get_dialect_visitor(sa_dialect, name):
+    """
+    Get the visitor implementation for the given dialect.
+
+    Finds the visitor implementation based on the dialect class and
+    returns and instance initialized with the given name.
+
+    Binds dialect specific preparer to visitor.
+    """
+
+    # map sa dialect to migrate dialect and return visitor
+    sa_dialect_name = getattr(sa_dialect, 'name', 'default')
+    migrate_dialect_cls = DIALECTS[sa_dialect_name]
+    visitor = getattr(migrate_dialect_cls, name)
+
+    # bind preparer
+    visitor.preparer = sa_dialect.preparer(sa_dialect)
+
+    return visitor
+
+def run_single_visitor(engine, visitorcallable, element,
+    connection=None, **kwargs):
+    """Taken from :meth:`sqlalchemy.engine.base.Engine._run_single_visitor`
+    with support for migrate visitors.
+    """
+    if connection is None:
+        conn = engine.contextual_connect(close_with_result=False)
+    else:
+        conn = connection
+    visitor = visitorcallable(engine.dialect, conn)
+    try:
+        if hasattr(element, '__migrate_visit_name__'):
+            fn = getattr(visitor, 'visit_' + element.__migrate_visit_name__)
+        else:
+            fn = getattr(visitor, 'visit_' + element.__visit_name__)
+        fn(element, **kwargs)
+    finally:
+        if connection is None:
+            conn.close()

rhodecode/lib/dbmigrate/migrate/changeset/schema.py

@@ -0,0 +1,669
+"""
+   Schema module providing common schema operations.
+"""
+import warnings
+
+from UserDict import DictMixin
+
+import sqlalchemy
+
+from sqlalchemy.schema import ForeignKeyConstraint
+from sqlalchemy.schema import UniqueConstraint
+
+from rhodecode.lib.dbmigrate.migrate.exceptions import *
+from rhodecode.lib.dbmigrate.migrate.changeset import SQLA_06
+from rhodecode.lib.dbmigrate.migrate.changeset.databases.visitor import (get_engine_visitor,
+                                                 run_single_visitor)
+
+
+__all__ = [
+    'create_column',
+    'drop_column',
+    'alter_column',
+    'rename_table',
+    'rename_index',
+    'ChangesetTable',
+    'ChangesetColumn',
+    'ChangesetIndex',
+    'ChangesetDefaultClause',
+    'ColumnDelta',
+]
+
+DEFAULT_ALTER_METADATA = True
+
+
+def create_column(column, table=None, *p, **kw):
+    """Create a column, given the table.
+    
+    API to :meth:`ChangesetColumn.create`.
+    """
+    if table is not None:
+        return table.create_column(column, *p, **kw)
+    return column.create(*p, **kw)
+
+
+def drop_column(column, table=None, *p, **kw):
+    """Drop a column, given the table.
+    
+    API to :meth:`ChangesetColumn.drop`.
+    """
+    if table is not None:
+        return table.drop_column(column, *p, **kw)
+    return column.drop(*p, **kw)
+
+
+def rename_table(table, name, engine=None, **kw):
+    """Rename a table.
+
+    If Table instance is given, engine is not used.
+
+    API to :meth:`ChangesetTable.rename`.
+
+    :param table: Table to be renamed.
+    :param name: New name for Table.
+    :param engine: Engine instance.
+    :type table: string or Table instance
+    :type name: string
+    :type engine: obj
+    """
+    table = _to_table(table, engine)
+    table.rename(name, **kw)
+
+
+def rename_index(index, name, table=None, engine=None, **kw):
+    """Rename an index.
+
+    If Index instance is given,
+    table and engine are not used.
+
+    API to :meth:`ChangesetIndex.rename`.
+
+    :param index: Index to be renamed.
+    :param name: New name for index.
+    :param table: Table to which Index is reffered.
+    :param engine: Engine instance.
+    :type index: string or Index instance
+    :type name: string
+    :type table: string or Table instance
+    :type engine: obj
+    """
+    index = _to_index(index, table, engine)
+    index.rename(name, **kw)
+
+
+def alter_column(*p, **k):
+    """Alter a column.
+
+    This is a helper function that creates a :class:`ColumnDelta` and
+    runs it.
+
+    :argument column:
+      The name of the column to be altered or a
+      :class:`ChangesetColumn` column representing it.
+
+    :param table:
+      A :class:`~sqlalchemy.schema.Table` or table name to
+      for the table where the column will be changed.
+
+    :param engine:
+      The :class:`~sqlalchemy.engine.base.Engine` to use for table
+      reflection and schema alterations.
+    
+    :param alter_metadata:
+      If `True`, which is the default, the
+      :class:`~sqlalchemy.schema.Column` will also modified.
+      If `False`, the :class:`~sqlalchemy.schema.Column` will be left
+      as it was.
+    
+    :returns: A :class:`ColumnDelta` instance representing the change.
+
+    
+    """
+
+    k.setdefault('alter_metadata', DEFAULT_ALTER_METADATA)
+
+    if 'table' not in k and isinstance(p[0], sqlalchemy.Column):
+        k['table'] = p[0].table
+    if 'engine' not in k:
+        k['engine'] = k['table'].bind
+
+    # deprecation
+    if len(p) >= 2 and isinstance(p[1], sqlalchemy.Column):
+        warnings.warn(
+            "Passing a Column object to alter_column is deprecated."
+            " Just pass in keyword parameters instead.",
+            MigrateDeprecationWarning
+            )
+    engine = k['engine']
+    delta = ColumnDelta(*p, **k)
+
+    visitorcallable = get_engine_visitor(engine, 'schemachanger')
+    engine._run_visitor(visitorcallable, delta)
+
+    return delta
+
+
+def _to_table(table, engine=None):
+    """Return if instance of Table, else construct new with metadata"""
+    if isinstance(table, sqlalchemy.Table):
+        return table
+
+    # Given: table name, maybe an engine
+    meta = sqlalchemy.MetaData()
+    if engine is not None:
+        meta.bind = engine
+    return sqlalchemy.Table(table, meta)
+
+
+def _to_index(index, table=None, engine=None):
+    """Return if instance of Index, else construct new with metadata"""
+    if isinstance(index, sqlalchemy.Index):
+        return index
+
+    # Given: index name; table name required
+    table = _to_table(table, engine)
+    ret = sqlalchemy.Index(index)
+    ret.table = table
+    return ret
+
+
+class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem):
+    """Extracts the differences between two columns/column-parameters
+
+        May receive parameters arranged in several different ways:
+
+        * **current_column, new_column, \*p, \*\*kw**
+            Additional parameters can be specified to override column
+            differences.
+
+        * **current_column, \*p, \*\*kw**
+            Additional parameters alter current_column. Table name is extracted
+            from current_column object.
+            Name is changed to current_column.name from current_name,
+            if current_name is specified.
+
+        * **current_col_name, \*p, \*\*kw**
+            Table kw must specified.
+
+        :param table: Table at which current Column should be bound to.\
+        If table name is given, reflection will be used.
+        :type table: string or Table instance
+        :param alter_metadata: If True, it will apply changes to metadata.
+        :type alter_metadata: bool
+        :param metadata: If `alter_metadata` is true, \
+        metadata is used to reflect table names into
+        :type metadata: :class:`MetaData` instance
+        :param engine: When reflecting tables, either engine or metadata must \
+        be specified to acquire engine object.
+        :type engine: :class:`Engine` instance
+        :returns: :class:`ColumnDelta` instance provides interface for altered attributes to \
+        `result_column` through :func:`dict` alike object.
+
+        * :class:`ColumnDelta`.result_column is altered column with new attributes
+
+        * :class:`ColumnDelta`.current_name is current name of column in db
+
+
+    """
+
+    # Column attributes that can be altered
+    diff_keys = ('name', 'type', 'primary_key', 'nullable',
+        'server_onupdate', 'server_default', 'autoincrement')
+    diffs = dict()
+    __visit_name__ = 'column'
+
+    def __init__(self, *p, **kw):
+        self.alter_metadata = kw.pop("alter_metadata", False)
+        self.meta = kw.pop("metadata", None)
+        self.engine = kw.pop("engine", None)
+
+        # Things are initialized differently depending on how many column
+        # parameters are given. Figure out how many and call the appropriate
+        # method.
+        if len(p) >= 1 and isinstance(p[0], sqlalchemy.Column):
+            # At least one column specified
+            if len(p) >= 2 and isinstance(p[1], sqlalchemy.Column):
+                # Two columns specified
+                diffs = self.compare_2_columns(*p, **kw)
+            else:
+                # Exactly one column specified
+                diffs = self.compare_1_column(*p, **kw)
+        else:
+            # Zero columns specified
+            if not len(p) or not isinstance(p[0], basestring):
+                raise ValueError("First argument must be column name")
+            diffs = self.compare_parameters(*p, **kw)
+
+        self.apply_diffs(diffs)
+
+    def __repr__(self):
+        return '<ColumnDelta altermetadata=%r, %s>' % (self.alter_metadata,
+            super(ColumnDelta, self).__repr__())
+
+    def __getitem__(self, key):
+        if key not in self.keys():
+            raise KeyError("No such diff key, available: %s" % self.diffs)
+        return getattr(self.result_column, key)
+
+    def __setitem__(self, key, value):
+        if key not in self.keys():
+            raise KeyError("No such diff key, available: %s" % self.diffs)
+        setattr(self.result_column, key, value)
+
+    def __delitem__(self, key):
+        raise NotImplementedError
+
+    def keys(self):
+        return self.diffs.keys()
+
+    def compare_parameters(self, current_name, *p, **k):
+        """Compares Column objects with reflection"""
+        self.table = k.pop('table')
+        self.result_column = self._table.c.get(current_name)
+        if len(p):
+            k = self._extract_parameters(p, k, self.result_column)
+        return k
+
+    def compare_1_column(self, col, *p, **k):
+        """Compares one Column object"""
+        self.table = k.pop('table', None)
+        if self.table is None:
+            self.table = col.table
+        self.result_column = col
+        if len(p):
+            k = self._extract_parameters(p, k, self.result_column)
+        return k
+
+    def compare_2_columns(self, old_col, new_col, *p, **k):
+        """Compares two Column objects"""
+        self.process_column(new_col)
+        self.table = k.pop('table', None)
+        # we cannot use bool() on table in SA06 
+        if self.table is None:
+            self.table = old_col.table
+        if self.table is None:
+            new_col.table
+        self.result_column = old_col
+
+        # set differences
+        # leave out some stuff for later comp
+        for key in (set(self.diff_keys) - set(('type',))):
+            val = getattr(new_col, key, None)
+            if getattr(self.result_column, key, None) != val:
+                k.setdefault(key, val)
+
+        # inspect types
+        if not self.are_column_types_eq(self.result_column.type, new_col.type):
+            k.setdefault('type', new_col.type)
+
+        if len(p):
+            k = self._extract_parameters(p, k, self.result_column)
+        return k
+
+    def apply_diffs(self, diffs):
+        """Populate dict and column object with new values"""
+        self.diffs = diffs
+        for key in self.diff_keys:
+            if key in diffs:
+                setattr(self.result_column, key, diffs[key])
+
+        self.process_column(self.result_column)
+
+        # create an instance of class type if not yet
+        if 'type' in diffs and callable(self.result_column.type):
+            self.result_column.type = self.result_column.type()
+
+        # add column to the table
+        if self.table is not None and self.alter_metadata:
+            self.result_column.add_to_table(self.table)
+
+    def are_column_types_eq(self, old_type, new_type):
+        """Compares two types to be equal"""
+        ret = old_type.__class__ == new_type.__class__
+
+        # String length is a special case
+        if ret and isinstance(new_type, sqlalchemy.types.String):
+            ret = (getattr(old_type, 'length', None) == \
+                       getattr(new_type, 'length', None))
+        return ret
+
+    def _extract_parameters(self, p, k, column):
+        """Extracts data from p and modifies diffs"""
+        p = list(p)
+        while len(p):
+            if isinstance(p[0], basestring):
+                k.setdefault('name', p.pop(0))
+            elif isinstance(p[0], sqlalchemy.types.AbstractType):
+                k.setdefault('type', p.pop(0))
+            elif callable(p[0]):
+                p[0] = p[0]()
+            else:
+                break
+
+        if len(p):
+            new_col = column.copy_fixed()
+            new_col._init_items(*p)
+            k = self.compare_2_columns(column, new_col, **k)
+        return k
+
+    def process_column(self, column):
+        """Processes default values for column"""
+        # XXX: this is a snippet from SA processing of positional parameters
+        if not SQLA_06 and column.args:
+            toinit = list(column.args)
+        else:
+            toinit = list()
+
+        if column.server_default is not None:
+            if isinstance(column.server_default, sqlalchemy.FetchedValue):
+                toinit.append(column.server_default)
+            else:
+                toinit.append(sqlalchemy.DefaultClause(column.server_default))
+        if column.server_onupdate is not None:
+            if isinstance(column.server_onupdate, FetchedValue):
+                toinit.append(column.server_default)
+            else:
+                toinit.append(sqlalchemy.DefaultClause(column.server_onupdate,
+                                            for_update=True))
+        if toinit:
+            column._init_items(*toinit)
+
+        if not SQLA_06:
+            column.args = []
+
+    def _get_table(self):
+        return getattr(self, '_table', None)
+
+    def _set_table(self, table):
+        if isinstance(table, basestring):
+            if self.alter_metadata:
+                if not self.meta:
+                    raise ValueError("metadata must be specified for table"
+                        " reflection when using alter_metadata")
+                meta = self.meta
+                if self.engine:
+                    meta.bind = self.engine
+            else:
+                if not self.engine and not self.meta:
+                    raise ValueError("engine or metadata must be specified"
+                        " to reflect tables")
+                if not self.engine:
+                    self.engine = self.meta.bind
+                meta = sqlalchemy.MetaData(bind=self.engine)
+            self._table = sqlalchemy.Table(table, meta, autoload=True)
+        elif isinstance(table, sqlalchemy.Table):
+            self._table = table
+            if not self.alter_metadata:
+                self._table.meta = sqlalchemy.MetaData(bind=self._table.bind)
+
+    def _get_result_column(self):
+        return getattr(self, '_result_column', None)
+
+    def _set_result_column(self, column):
+        """Set Column to Table based on alter_metadata evaluation."""
+        self.process_column(column)
+        if not hasattr(self, 'current_name'):
+            self.current_name = column.name
+        if self.alter_metadata:
+            self._result_column = column
+        else:
+            self._result_column = column.copy_fixed()
+
+    table = property(_get_table, _set_table)
+    result_column = property(_get_result_column, _set_result_column)
+
+
+class ChangesetTable(object):
+    """Changeset extensions to SQLAlchemy tables."""
+
+    def create_column(self, column, *p, **kw):
+        """Creates a column.
+
+        The column parameter may be a column definition or the name of
+        a column in this table.
+
+        API to :meth:`ChangesetColumn.create`
+
+        :param column: Column to be created
+        :type column: Column instance or string
+        """
+        if not isinstance(column, sqlalchemy.Column):
+            # It's a column name
+            column = getattr(self.c, str(column))
+        column.create(table=self, *p, **kw)
+
+    def drop_column(self, column, *p, **kw):
+        """Drop a column, given its name or definition.
+
+        API to :meth:`ChangesetColumn.drop`
+
+        :param column: Column to be droped
+        :type column: Column instance or string
+        """
+        if not isinstance(column, sqlalchemy.Column):
+            # It's a column name
+            try:
+                column = getattr(self.c, str(column))
+            except AttributeError:
+                # That column isn't part of the table. We don't need
+                # its entire definition to drop the column, just its
+                # name, so create a dummy column with the same name.
+                column = sqlalchemy.Column(str(column), sqlalchemy.Integer())
+        column.drop(table=self, *p, **kw)
+
+    def rename(self, name, connection=None, **kwargs):
+        """Rename this table.
+
+        :param name: New name of the table.
+        :type name: string
+        :param alter_metadata: If True, table will be removed from metadata
+        :type alter_metadata: bool
+        :param connection: reuse connection istead of creating new one.
+        :type connection: :class:`sqlalchemy.engine.base.Connection` instance
+        """
+        self.alter_metadata = kwargs.pop('alter_metadata', DEFAULT_ALTER_METADATA)
+        engine = self.bind
+        self.new_name = name
+        visitorcallable = get_engine_visitor(engine, 'schemachanger')
+        run_single_visitor(engine, visitorcallable, self, connection, **kwargs)
+
+        # Fix metadata registration
+        if self.alter_metadata:
+            self.name = name
+            self.deregister()
+            self._set_parent(self.metadata)
+
+    def _meta_key(self):
+        return sqlalchemy.schema._get_table_key(self.name, self.schema)
+
+    def deregister(self):
+        """Remove this table from its metadata"""
+        key = self._meta_key()
+        meta = self.metadata
+        if key in meta.tables:
+            del meta.tables[key]
+
+
+class ChangesetColumn(object):
+    """Changeset extensions to SQLAlchemy columns."""
+
+    def alter(self, *p, **k):
+        """Makes a call to :func:`alter_column` for the column this
+        method is called on. 
+        """
+        if 'table' not in k:
+            k['table'] = self.table
+        if 'engine' not in k:
+            k['engine'] = k['table'].bind
+        return alter_column(self, *p, **k)
+
+    def create(self, table=None, index_name=None, unique_name=None,
+               primary_key_name=None, populate_default=True, connection=None, **kwargs):
+        """Create this column in the database.
+
+        Assumes the given table exists. ``ALTER TABLE ADD COLUMN``,
+        for most databases.
+
+        :param table: Table instance to create on.
+        :param index_name: Creates :class:`ChangesetIndex` on this column.
+        :param unique_name: Creates :class:\
+`~migrate.changeset.constraint.UniqueConstraint` on this column.
+        :param primary_key_name: Creates :class:\
+`~migrate.changeset.constraint.PrimaryKeyConstraint` on this column.
+        :param alter_metadata: If True, column will be added to table object.
+        :param populate_default: If True, created column will be \
+populated with defaults
+        :param connection: reuse connection istead of creating new one.
+        :type table: Table instance
+        :type index_name: string
+        :type unique_name: string
+        :type primary_key_name: string
+        :type alter_metadata: bool
+        :type populate_default: bool
+        :type connection: :class:`sqlalchemy.engine.base.Connection` instance
+
+        :returns: self
+        """
+        self.populate_default = populate_default
+        self.alter_metadata = kwargs.pop('alter_metadata', DEFAULT_ALTER_METADATA)
+        self.index_name = index_name
+        self.unique_name = unique_name
+        self.primary_key_name = primary_key_name
+        for cons in ('index_name', 'unique_name', 'primary_key_name'):
+            self._check_sanity_constraints(cons)
+
+        if self.alter_metadata:
+            self.add_to_table(table)
+        engine = self.table.bind
+        visitorcallable = get_engine_visitor(engine, 'columngenerator')
+        engine._run_visitor(visitorcallable, self, connection, **kwargs)
+
+        # TODO: reuse existing connection
+        if self.populate_default and self.default is not None:
+            stmt = table.update().values({self: engine._execute_default(self.default)})
+            engine.execute(stmt)
+
+        return self
+
+    def drop(self, table=None, connection=None, **kwargs):
+        """Drop this column from the database, leaving its table intact.
+
+        ``ALTER TABLE DROP COLUMN``, for most databases.
+
+        :param alter_metadata: If True, column will be removed from table object.
+        :type alter_metadata: bool
+        :param connection: reuse connection istead of creating new one.
+        :type connection: :class:`sqlalchemy.engine.base.Connection` instance
+        """
+        self.alter_metadata = kwargs.pop('alter_metadata', DEFAULT_ALTER_METADATA)
+        if table is not None:
+            self.table = table
+        engine = self.table.bind
+        if self.alter_metadata:
+            self.remove_from_table(self.table, unset_table=False)
+        visitorcallable = get_engine_visitor(engine, 'columndropper')
+        engine._run_visitor(visitorcallable, self, connection, **kwargs)
+        if self.alter_metadata:
+            self.table = None
+        return self
+
+    def add_to_table(self, table):
+        if table is not None  and self.table is None:
+            self._set_parent(table)
+
+    def _col_name_in_constraint(self, cons, name):
+        return False
+
+    def remove_from_table(self, table, unset_table=True):
+        # TODO: remove primary keys, constraints, etc
+        if unset_table:
+            self.table = None
+
+        to_drop = set()
+        for index in table.indexes:
+            columns = []
+            for col in index.columns:
+                if col.name != self.name:
+                    columns.append(col)
+            if columns:
+                index.columns = columns
+            else:
+                to_drop.add(index)
+        table.indexes = table.indexes - to_drop
+
+        to_drop = set()
+        for cons in table.constraints:
+            # TODO: deal with other types of constraint
+            if isinstance(cons, (ForeignKeyConstraint,
+                                UniqueConstraint)):
+                for col_name in cons.columns:
+                    if not isinstance(col_name, basestring):
+                        col_name = col_name.name
+                    if self.name == col_name:
+                        to_drop.add(cons)
+        table.constraints = table.constraints - to_drop
+
+        if table.c.contains_column(self):
+            table.c.remove(self)
+
+    # TODO: this is fixed in 0.6
+    def copy_fixed(self, **kw):
+        """Create a copy of this ``Column``, with all attributes."""
+        return sqlalchemy.Column(self.name, self.type, self.default,
+            key=self.key,
+            primary_key=self.primary_key,
+            nullable=self.nullable,
+            quote=self.quote,
+            index=self.index,
+            unique=self.unique,
+            onupdate=self.onupdate,
+            autoincrement=self.autoincrement,
+            server_default=self.server_default,
+            server_onupdate=self.server_onupdate,
+            *[c.copy(**kw) for c in self.constraints])
+
+    def _check_sanity_constraints(self, name):
+        """Check if constraints names are correct"""
+        obj = getattr(self, name)
+        if (getattr(self, name[:-5]) and not obj):
+            raise InvalidConstraintError("Column.create() accepts index_name,"
+            " primary_key_name and unique_name to generate constraints")
+        if not isinstance(obj, basestring) and obj is not None:
+            raise InvalidConstraintError(
+            "%s argument for column must be constraint name" % name)
+
+
+class ChangesetIndex(object):
+    """Changeset extensions to SQLAlchemy Indexes."""
+
+    __visit_name__ = 'index'
+
+    def rename(self, name, connection=None, **kwargs):
+        """Change the name of an index.
+
+        :param name: New name of the Index.
+        :type name: string
+        :param alter_metadata: If True, Index object will be altered.
+        :type alter_metadata: bool
+        :param connection: reuse connection istead of creating new one.
+        :type connection: :class:`sqlalchemy.engine.base.Connection` instance
+        """
+        self.alter_metadata = kwargs.pop('alter_metadata', DEFAULT_ALTER_METADATA)
+        engine = self.table.bind
+        self.new_name = name
+        visitorcallable = get_engine_visitor(engine, 'schemachanger')
+        engine._run_visitor(visitorcallable, self, connection, **kwargs)
+        if self.alter_metadata:
+            self.name = name
+
+
+class ChangesetDefaultClause(object):
+    """Implements comparison between :class:`DefaultClause` instances"""
+
+    def __eq__(self, other):
+        if isinstance(other, self.__class__):
+            if self.arg == other.arg:
+                return True
+
+    def __ne__(self, other):
+        return not self.__eq__(other)

rhodecode/lib/dbmigrate/migrate/exceptions.py

@@ -0,0 +1,87
+"""
+   Provide exception classes for :mod:`migrate`
+"""
+
+
+class Error(Exception):
+    """Error base class."""
+
+
+class ApiError(Error):
+    """Base class for API errors."""
+
+
+class KnownError(ApiError):
+    """A known error condition."""
+
+
+class UsageError(ApiError):
+    """A known error condition where help should be displayed."""
+
+
+class ControlledSchemaError(Error):
+    """Base class for controlled schema errors."""
+
+
+class InvalidVersionError(ControlledSchemaError):
+    """Invalid version number."""
+
+
+class DatabaseNotControlledError(ControlledSchemaError):
+    """Database should be under version control, but it's not."""
+
+
+class DatabaseAlreadyControlledError(ControlledSchemaError):
+    """Database shouldn't be under version control, but it is"""
+
+
+class WrongRepositoryError(ControlledSchemaError):
+    """This database is under version control by another repository."""
+
+
+class NoSuchTableError(ControlledSchemaError):
+    """The table does not exist."""
+
+
+class PathError(Error):
+    """Base class for path errors."""
+
+
+class PathNotFoundError(PathError):
+    """A path with no file was required; found a file."""
+
+
+class PathFoundError(PathError):
+    """A path with a file was required; found no file."""
+
+
+class RepositoryError(Error):
+    """Base class for repository errors."""
+
+
+class InvalidRepositoryError(RepositoryError):
+    """Invalid repository error."""
+
+
+class ScriptError(Error):
+    """Base class for script errors."""
+
+
+class InvalidScriptError(ScriptError):
+    """Invalid script error."""
+
+
+class InvalidVersionError(Error):
+    """Invalid version error."""
+
+# migrate.changeset
+
+class NotSupportedError(Error):
+    """Not supported error"""
+
+
+class InvalidConstraintError(Error):
+    """Invalid constraint error"""
+
+class MigrateDeprecationWarning(DeprecationWarning):
+    """Warning for deprecated features in Migrate"""

rhodecode/lib/dbmigrate/migrate/versioning/__init__.py

@@ -0,0 +1,5
+"""
+   This package provides functionality to create and manage
+   repositories of database schema changesets and to apply these
+   changesets to databases.
+"""

rhodecode/lib/dbmigrate/migrate/versioning/api.py

@@ -0,0 +1,383
+"""
+   This module provides an external API to the versioning system.
+
+   .. versionchanged:: 0.6.0
+    :func:`migrate.versioning.api.test` and schema diff functions
+    changed order of positional arguments so all accept `url` and `repository`
+    as first arguments.
+
+   .. versionchanged:: 0.5.4
+    ``--preview_sql`` displays source file when using SQL scripts.
+    If Python script is used, it runs the action with mocked engine and
+    returns captured SQL statements.
+
+   .. versionchanged:: 0.5.4
+    Deprecated ``--echo`` parameter in favour of new
+    :func:`migrate.versioning.util.construct_engine` behavior.
+"""
+
+# Dear migrate developers,
+#
+# please do not comment this module using sphinx syntax because its
+# docstrings are presented as user help and most users cannot
+# interpret sphinx annotated ReStructuredText.
+#
+# Thanks,
+# Jan Dittberner
+
+import sys
+import inspect
+import logging
+
+from rhodecode.lib.dbmigrate.migrate import exceptions
+from rhodecode.lib.dbmigrate.migrate.versioning import repository, schema, version, \
+    script as script_ # command name conflict
+from rhodecode.lib.dbmigrate.migrate.versioning.util import catch_known_errors, with_engine
+
+
+log = logging.getLogger(__name__)
+command_desc = {
+    'help': 'displays help on a given command',
+    'create': 'create an empty repository at the specified path',
+    'script': 'create an empty change Python script',
+    'script_sql': 'create empty change SQL scripts for given database',
+    'version': 'display the latest version available in a repository',
+    'db_version': 'show the current version of the repository under version control',
+    'source': 'display the Python code for a particular version in this repository',
+    'version_control': 'mark a database as under this repository\'s version control',
+    'upgrade': 'upgrade a database to a later version',
+    'downgrade': 'downgrade a database to an earlier version',
+    'drop_version_control': 'removes version control from a database',
+    'manage': 'creates a Python script that runs Migrate with a set of default values',
+    'test': 'performs the upgrade and downgrade command on the given database',
+    'compare_model_to_db': 'compare MetaData against the current database state',
+    'create_model': 'dump the current database as a Python model to stdout',
+    'make_update_script_for_model': 'create a script changing the old MetaData to the new (current) MetaData',
+    'update_db_from_model': 'modify the database to match the structure of the current MetaData',
+}
+__all__ = command_desc.keys()
+
+Repository = repository.Repository
+ControlledSchema = schema.ControlledSchema
+VerNum = version.VerNum
+PythonScript = script_.PythonScript
+SqlScript = script_.SqlScript
+
+
+# deprecated
+def help(cmd=None, **opts):
+    """%prog help COMMAND
+
+    Displays help on a given command.
+    """
+    if cmd is None:
+        raise exceptions.UsageError(None)
+    try:
+        func = globals()[cmd]
+    except:
+        raise exceptions.UsageError(
+            "'%s' isn't a valid command. Try 'help COMMAND'" % cmd)
+    ret = func.__doc__
+    if sys.argv[0]:
+        ret = ret.replace('%prog', sys.argv[0])
+    return ret
+
+@catch_known_errors
+def create(repository, name, **opts):
+    """%prog create REPOSITORY_PATH NAME [--table=TABLE]
+
+    Create an empty repository at the specified path.
+
+    You can specify the version_table to be used; by default, it is
+    'migrate_version'.  This table is created in all version-controlled
+    databases.
+    """
+    repo_path = Repository.create(repository, name, **opts)
+
+
+@catch_known_errors
+def script(description, repository, **opts):
+    """%prog script DESCRIPTION REPOSITORY_PATH
+
+    Create an empty change script using the next unused version number
+    appended with the given description.
+
+    For instance, manage.py script "Add initial tables" creates:
+    repository/versions/001_Add_initial_tables.py
+    """
+    repo = Repository(repository)
+    repo.create_script(description, **opts)
+
+
+@catch_known_errors
+def script_sql(database, repository, **opts):
+    """%prog script_sql DATABASE REPOSITORY_PATH
+
+    Create empty change SQL scripts for given DATABASE, where DATABASE
+    is either specific ('postgres', 'mysql', 'oracle', 'sqlite', etc.)
+    or generic ('default').
+
+    For instance, manage.py script_sql postgres creates:
+    repository/versions/001_postgres_upgrade.sql and
+    repository/versions/001_postgres_postgres.sql
+    """
+    repo = Repository(repository)
+    repo.create_script_sql(database, **opts)
+
+
+def version(repository, **opts):
+    """%prog version REPOSITORY_PATH
+
+    Display the latest version available in a repository.
+    """
+    repo = Repository(repository)
+    return repo.latest
+
+
+@with_engine
+def db_version(url, repository, **opts):
+    """%prog db_version URL REPOSITORY_PATH
+
+    Show the current version of the repository with the given
+    connection string, under version control of the specified
+    repository.
+
+    The url should be any valid SQLAlchemy connection string.
+    """
+    engine = opts.pop('engine')
+    schema = ControlledSchema(engine, repository)
+    return schema.version
+
+
+def source(version, dest=None, repository=None, **opts):
+    """%prog source VERSION [DESTINATION] --repository=REPOSITORY_PATH
+
+    Display the Python code for a particular version in this
+    repository.  Save it to the file at DESTINATION or, if omitted,
+    send to stdout.
+    """
+    if repository is None:
+        raise exceptions.UsageError("A repository must be specified")
+    repo = Repository(repository)
+    ret = repo.version(version).script().source()
+    if dest is not None:
+        dest = open(dest, 'w')
+        dest.write(ret)
+        dest.close()
+        ret = None
+    return ret
+
+
+def upgrade(url, repository, version=None, **opts):
+    """%prog upgrade URL REPOSITORY_PATH [VERSION] [--preview_py|--preview_sql]
+
+    Upgrade a database to a later version.
+
+    This runs the upgrade() function defined in your change scripts.
+
+    By default, the database is updated to the latest available
+    version. You may specify a version instead, if you wish.
+
+    You may preview the Python or SQL code to be executed, rather than
+    actually executing it, using the appropriate 'preview' option.
+    """
+    err = "Cannot upgrade a database of version %s to version %s. "\
+        "Try 'downgrade' instead."
+    return _migrate(url, repository, version, upgrade=True, err=err, **opts)
+
+
+def downgrade(url, repository, version, **opts):
+    """%prog downgrade URL REPOSITORY_PATH VERSION [--preview_py|--preview_sql]
+
+    Downgrade a database to an earlier version.
+
+    This is the reverse of upgrade; this runs the downgrade() function
+    defined in your change scripts.
+
+    You may preview the Python or SQL code to be executed, rather than
+    actually executing it, using the appropriate 'preview' option.
+    """
+    err = "Cannot downgrade a database of version %s to version %s. "\
+        "Try 'upgrade' instead."
+    return _migrate(url, repository, version, upgrade=False, err=err, **opts)
+
+@with_engine
+def test(url, repository, **opts):
+    """%prog test URL REPOSITORY_PATH [VERSION]
+
+    Performs the upgrade and downgrade option on the given
+    database. This is not a real test and may leave the database in a
+    bad state. You should therefore better run the test on a copy of
+    your database.
+    """
+    engine = opts.pop('engine')
+    repos = Repository(repository)
+    script = repos.version(None).script()
+
+    # Upgrade
+    log.info("Upgrading...")
+    script.run(engine, 1)
+    log.info("done")
+
+    log.info("Downgrading...")
+    script.run(engine, -1)
+    log.info("done")
+    log.info("Success")
+
+
+@with_engine
+def version_control(url, repository, version=None, **opts):
+    """%prog version_control URL REPOSITORY_PATH [VERSION]
+
+    Mark a database as under this repository's version control.
+
+    Once a database is under version control, schema changes should
+    only be done via change scripts in this repository.
+
+    This creates the table version_table in the database.
+
+    The url should be any valid SQLAlchemy connection string.
+
+    By default, the database begins at version 0 and is assumed to be
+    empty.  If the database is not empty, you may specify a version at
+    which to begin instead. No attempt is made to verify this
+    version's correctness - the database schema is expected to be
+    identical to what it would be if the database were created from
+    scratch.
+    """
+    engine = opts.pop('engine')
+    ControlledSchema.create(engine, repository, version)
+
+
+@with_engine
+def drop_version_control(url, repository, **opts):
+    """%prog drop_version_control URL REPOSITORY_PATH
+
+    Removes version control from a database.
+    """
+    engine = opts.pop('engine')
+    schema = ControlledSchema(engine, repository)
+    schema.drop()
+
+
+def manage(file, **opts):
+    """%prog manage FILENAME [VARIABLES...]
+
+    Creates a script that runs Migrate with a set of default values.
+
+    For example::
+
+        %prog manage manage.py --repository=/path/to/repository \
+--url=sqlite:///project.db
+
+    would create the script manage.py. The following two commands
+    would then have exactly the same results::
+
+        python manage.py version
+        %prog version --repository=/path/to/repository
+    """
+    Repository.create_manage_file(file, **opts)
+
+
+@with_engine
+def compare_model_to_db(url, repository, model, **opts):
+    """%prog compare_model_to_db URL REPOSITORY_PATH MODEL
+
+    Compare the current model (assumed to be a module level variable
+    of type sqlalchemy.MetaData) against the current database.
+
+    NOTE: This is EXPERIMENTAL.
+    """  # TODO: get rid of EXPERIMENTAL label
+    engine = opts.pop('engine')
+    return ControlledSchema.compare_model_to_db(engine, model, repository)
+
+
+@with_engine
+def create_model(url, repository, **opts):
+    """%prog create_model URL REPOSITORY_PATH [DECLERATIVE=True]
+
+    Dump the current database as a Python model to stdout.
+
+    NOTE: This is EXPERIMENTAL.
+    """  # TODO: get rid of EXPERIMENTAL label
+    engine = opts.pop('engine')
+    declarative = opts.get('declarative', False)
+    return ControlledSchema.create_model(engine, repository, declarative)
+
+
+@catch_known_errors
+@with_engine
+def make_update_script_for_model(url, repository, oldmodel, model, **opts):
+    """%prog make_update_script_for_model URL OLDMODEL MODEL REPOSITORY_PATH
+
+    Create a script changing the old Python model to the new (current)
+    Python model, sending to stdout.
+
+    NOTE: This is EXPERIMENTAL.
+    """  # TODO: get rid of EXPERIMENTAL label
+    engine = opts.pop('engine')
+    return PythonScript.make_update_script_for_model(
+        engine, oldmodel, model, repository, **opts)
+
+
+@with_engine
+def update_db_from_model(url, repository, model, **opts):
+    """%prog update_db_from_model URL REPOSITORY_PATH MODEL
+
+    Modify the database to match the structure of the current Python
+    model. This also sets the db_version number to the latest in the
+    repository.
+
+    NOTE: This is EXPERIMENTAL.
+    """  # TODO: get rid of EXPERIMENTAL label
+    engine = opts.pop('engine')
+    schema = ControlledSchema(engine, repository)
+    schema.update_db_from_model(model)
+
+@with_engine
+def _migrate(url, repository, version, upgrade, err, **opts):
+    engine = opts.pop('engine')
+    url = str(engine.url)
+    schema = ControlledSchema(engine, repository)
+    version = _migrate_version(schema, version, upgrade, err)
+
+    changeset = schema.changeset(version)
+    for ver, change in changeset:
+        nextver = ver + changeset.step
+        log.info('%s -> %s... ', ver, nextver)
+
+        if opts.get('preview_sql'):
+            if isinstance(change, PythonScript):
+                log.info(change.preview_sql(url, changeset.step, **opts))
+            elif isinstance(change, SqlScript):
+                log.info(change.source())
+
+        elif opts.get('preview_py'):
+            if not isinstance(change, PythonScript):
+                raise exceptions.UsageError("Python source can be only displayed"
+                    " for python migration files")
+            source_ver = max(ver, nextver)
+            module = schema.repository.version(source_ver).script().module
+            funcname = upgrade and "upgrade" or "downgrade"
+            func = getattr(module, funcname)
+            log.info(inspect.getsource(func))
+        else:
+            schema.runchange(ver, change, changeset.step)
+            log.info('done')
+
+
+def _migrate_version(schema, version, upgrade, err):
+    if version is None:
+        return version
+    # Version is specified: ensure we're upgrading in the right direction
+    # (current version < target version for upgrading; reverse for down)
+    version = VerNum(version)
+    cur = schema.version
+    if upgrade is not None:
+        if upgrade:
+            direction = cur <= version
+        else:
+            direction = cur >= version
+        if not direction:
+            raise exceptions.KnownError(err % (cur, version))
+    return version

rhodecode/lib/dbmigrate/migrate/versioning/cfgparse.py

@@ -0,0 +1,27
+"""
+   Configuration parser module.
+"""
+
+from ConfigParser import ConfigParser
+
+from rhodecode.lib.dbmigrate.migrate.versioning.config import *
+from rhodecode.lib.dbmigrate.migrate.versioning import pathed
+
+
+class Parser(ConfigParser):
+    """A project configuration file."""
+
+    def to_dict(self, sections=None):
+        """It's easier to access config values like dictionaries"""
+        return self._sections
+
+
+class Config(pathed.Pathed, Parser):
+    """Configuration class."""
+
+    def __init__(self, path, *p, **k):
+        """Confirm the config file exists; read it."""
+        self.require_found(path)
+        pathed.Pathed.__init__(self, path)
+        Parser.__init__(self, *p, **k)
+        self.read(path)

rhodecode/lib/dbmigrate/migrate/versioning/config.py

@@ -0,0 +1,14
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+from sqlalchemy.util import OrderedDict
+
+
+__all__ = ['databases', 'operations']
+
+databases = ('sqlite', 'postgres', 'mysql', 'oracle', 'mssql', 'firebird')
+
+# Map operation names to function names
+operations = OrderedDict()
+operations['upgrade'] = 'upgrade'
+operations['downgrade'] = 'downgrade'

rhodecode/lib/dbmigrate/migrate/versioning/genmodel.py

@@ -0,0 +1,253
+"""
+   Code to generate a Python model from a database or differences
+   between a model and database.
+
+   Some of this is borrowed heavily from the AutoCode project at:
+   http://code.google.com/p/sqlautocode/
+"""
+
+import sys
+import logging
+
+import sqlalchemy
+
+from rhodecode.lib.dbmigrate import migrate
+from rhodecode.lib.dbmigrate.migrate import changeset
+
+log = logging.getLogger(__name__)
+HEADER = """
+## File autogenerated by genmodel.py
+
+from sqlalchemy import *
+meta = MetaData()
+"""
+
+DECLARATIVE_HEADER = """
+## File autogenerated by genmodel.py
+
+from sqlalchemy import *
+from sqlalchemy.ext import declarative
+
+Base = declarative.declarative_base()
+"""
+
+
+class ModelGenerator(object):
+
+    def __init__(self, diff, engine, declarative=False):
+        self.diff = diff
+        self.engine = engine
+        self.declarative = declarative
+
+    def column_repr(self, col):
+        kwarg = []
+        if col.key != col.name:
+            kwarg.append('key')
+        if col.primary_key:
+            col.primary_key = True  # otherwise it dumps it as 1
+            kwarg.append('primary_key')
+        if not col.nullable:
+            kwarg.append('nullable')
+        if col.onupdate:
+            kwarg.append('onupdate')
+        if col.default:
+            if col.primary_key:
+                # I found that PostgreSQL automatically creates a
+                # default value for the sequence, but let's not show
+                # that.
+                pass
+            else:
+                kwarg.append('default')
+        ks = ', '.join('%s=%r' % (k, getattr(col, k)) for k in kwarg)
+
+        # crs: not sure if this is good idea, but it gets rid of extra
+        # u''
+        name = col.name.encode('utf8')
+
+        type_ = col.type
+        for cls in col.type.__class__.__mro__:
+            if cls.__module__ == 'sqlalchemy.types' and \
+                not cls.__name__.isupper():
+                if cls is not type_.__class__:
+                    type_ = cls()
+                break
+
+        data = {
+            'name': name,
+            'type': type_,
+            'constraints': ', '.join([repr(cn) for cn in col.constraints]),
+            'args': ks and ks or ''}
+
+        if data['constraints']:
+            if data['args']:
+                data['args'] = ',' + data['args']
+
+        if data['constraints'] or data['args']:
+            data['maybeComma'] = ','
+        else:
+            data['maybeComma'] = ''
+
+        commonStuff = """ %(maybeComma)s %(constraints)s %(args)s)""" % data
+        commonStuff = commonStuff.strip()
+        data['commonStuff'] = commonStuff
+        if self.declarative:
+            return """%(name)s = Column(%(type)r%(commonStuff)s""" % data
+        else:
+            return """Column(%(name)r, %(type)r%(commonStuff)s""" % data
+
+    def getTableDefn(self, table):
+        out = []
+        tableName = table.name
+        if self.declarative:
+            out.append("class %(table)s(Base):" % {'table': tableName})
+            out.append("  __tablename__ = '%(table)s'" % {'table': tableName})
+            for col in table.columns:
+                out.append("  %s" % self.column_repr(col))
+        else:
+            out.append("%(table)s = Table('%(table)s', meta," % \
+                           {'table': tableName})
+            for col in table.columns:
+                out.append("  %s," % self.column_repr(col))
+            out.append(")")
+        return out
+
+    def _get_tables(self, missingA=False, missingB=False, modified=False):
+        to_process = []
+        for bool_, names, metadata in (
+            (missingA, self.diff.tables_missing_from_A, self.diff.metadataB),
+            (missingB, self.diff.tables_missing_from_B, self.diff.metadataA),
+            (modified, self.diff.tables_different, self.diff.metadataA),
+                ):
+            if bool_:
+                for name in names:
+                    yield metadata.tables.get(name)
+
+    def toPython(self):
+        """Assume database is current and model is empty."""
+        out = []
+        if self.declarative:
+            out.append(DECLARATIVE_HEADER)
+        else:
+            out.append(HEADER)
+        out.append("")
+        for table in self._get_tables(missingA=True):
+            out.extend(self.getTableDefn(table))
+            out.append("")
+        return '\n'.join(out)
+
+    def toUpgradeDowngradePython(self, indent='    '):
+        ''' Assume model is most current and database is out-of-date. '''
+        decls = ['from rhodecode.lib.dbmigrate.migrate.changeset import schema',
+                 'meta = MetaData()']
+        for table in self._get_tables(
+            missingA=True, missingB=True, modified=True
+            ):
+            decls.extend(self.getTableDefn(table))
+
+        upgradeCommands, downgradeCommands = [], []
+        for tableName in self.diff.tables_missing_from_A:
+            upgradeCommands.append("%(table)s.drop()" % {'table': tableName})
+            downgradeCommands.append("%(table)s.create()" % \
+                                         {'table': tableName})
+        for tableName in self.diff.tables_missing_from_B:
+            upgradeCommands.append("%(table)s.create()" % {'table': tableName})
+            downgradeCommands.append("%(table)s.drop()" % {'table': tableName})
+
+        for tableName in self.diff.tables_different:
+            dbTable = self.diff.metadataB.tables[tableName]
+            missingInDatabase, missingInModel, diffDecl = \
+                self.diff.colDiffs[tableName]
+            for col in missingInDatabase:
+                upgradeCommands.append('%s.columns[%r].create()' % (
+                        modelTable, col.name))
+                downgradeCommands.append('%s.columns[%r].drop()' % (
+                        modelTable, col.name))
+            for col in missingInModel:
+                upgradeCommands.append('%s.columns[%r].drop()' % (
+                        modelTable, col.name))
+                downgradeCommands.append('%s.columns[%r].create()' % (
+                        modelTable, col.name))
+            for modelCol, databaseCol, modelDecl, databaseDecl in diffDecl:
+                upgradeCommands.append(
+                    'assert False, "Can\'t alter columns: %s:%s=>%s"',
+                    modelTable, modelCol.name, databaseCol.name)
+                downgradeCommands.append(
+                    'assert False, "Can\'t alter columns: %s:%s=>%s"',
+                    modelTable, modelCol.name, databaseCol.name)
+        pre_command = '    meta.bind = migrate_engine'
+
+        return (
+            '\n'.join(decls),
+            '\n'.join([pre_command] + ['%s%s' % (indent, line) for line in upgradeCommands]),
+            '\n'.join([pre_command] + ['%s%s' % (indent, line) for line in downgradeCommands]))
+
+    def _db_can_handle_this_change(self, td):
+        if (td.columns_missing_from_B
+            and not td.columns_missing_from_A
+            and not td.columns_different):
+            # Even sqlite can handle this.
+            return True
+        else:
+            return not self.engine.url.drivername.startswith('sqlite')
+
+    def applyModel(self):
+        """Apply model to current database."""
+
+        meta = sqlalchemy.MetaData(self.engine)
+
+        for table in self._get_tables(missingA=True):
+            table = table.tometadata(meta)
+            table.drop()
+        for table in self._get_tables(missingB=True):
+            table = table.tometadata(meta)
+            table.create()
+        for modelTable in self._get_tables(modified=True):
+            tableName = modelTable.name
+            modelTable = modelTable.tometadata(meta)
+            dbTable = self.diff.metadataB.tables[tableName]
+
+            td = self.diff.tables_different[tableName]
+
+            if self._db_can_handle_this_change(td):
+
+                for col in td.columns_missing_from_B:
+                    modelTable.columns[col].create()
+                for col in td.columns_missing_from_A:
+                    dbTable.columns[col].drop()
+                # XXX handle column changes here.
+            else:
+                # Sqlite doesn't support drop column, so you have to
+                # do more: create temp table, copy data to it, drop
+                # old table, create new table, copy data back.
+                #
+                # I wonder if this is guaranteed to be unique?
+                tempName = '_temp_%s' % modelTable.name
+
+                def getCopyStatement():
+                    preparer = self.engine.dialect.preparer
+                    commonCols = []
+                    for modelCol in modelTable.columns:
+                        if modelCol.name in dbTable.columns:
+                            commonCols.append(modelCol.name)
+                    commonColsStr = ', '.join(commonCols)
+                    return 'INSERT INTO %s (%s) SELECT %s FROM %s' % \
+                        (tableName, commonColsStr, commonColsStr, tempName)
+
+                # Move the data in one transaction, so that we don't
+                # leave the database in a nasty state.
+                connection = self.engine.connect()
+                trans = connection.begin()
+                try:
+                    connection.execute(
+                        'CREATE TEMPORARY TABLE %s as SELECT * from %s' % \
+                            (tempName, modelTable.name))
+                    # make sure the drop takes place inside our
+                    # transaction with the bind parameter
+                    modelTable.drop(bind=connection)
+                    modelTable.create(bind=connection)
+                    connection.execute(getCopyStatement())
+                    connection.execute('DROP TABLE %s' % tempName)
+                    trans.commit()
+                except:
+                    trans.rollback()
+                    raise

rhodecode/lib/dbmigrate/migrate/versioning/migrate_repository.py

@@ -0,0 +1,100
+"""
+   Script to migrate repository from sqlalchemy <= 0.4.4 to the new
+   repository schema. This shouldn't use any other migrate modules, so
+   that it can work in any version.
+"""
+
+import os
+import sys
+import logging
+
+log = logging.getLogger(__name__)
+
+
+def usage():
+    """Gives usage information."""
+    print """Usage: %(prog)s repository-to-migrate
+
+    Upgrade your repository to the new flat format.
+
+    NOTE: You should probably make a backup before running this.
+    """ % {'prog': sys.argv[0]}
+
+    sys.exit(1)
+
+
+def delete_file(filepath):
+    """Deletes a file and prints a message."""
+    log.info('Deleting file: %s' % filepath)
+    os.remove(filepath)
+
+
+def move_file(src, tgt):
+    """Moves a file and prints a message."""
+    log.info('Moving file %s to %s' % (src, tgt))
+    if os.path.exists(tgt):
+        raise Exception(
+            'Cannot move file %s because target %s already exists' % \
+                (src, tgt))
+    os.rename(src, tgt)
+
+
+def delete_directory(dirpath):
+    """Delete a directory and print a message."""
+    log.info('Deleting directory: %s' % dirpath)
+    os.rmdir(dirpath)
+
+
+def migrate_repository(repos):
+    """Does the actual migration to the new repository format."""
+    log.info('Migrating repository at: %s to new format' % repos)
+    versions = '%s/versions' % repos
+    dirs = os.listdir(versions)
+    # Only use int's in list.
+    numdirs = [int(dirname) for dirname in dirs if dirname.isdigit()]
+    numdirs.sort()  # Sort list.
+    for dirname in numdirs:
+        origdir = '%s/%s' % (versions, dirname)
+        log.info('Working on directory: %s' % origdir)
+        files = os.listdir(origdir)
+        files.sort()
+        for filename in files:
+            # Delete compiled Python files.
+            if filename.endswith('.pyc') or filename.endswith('.pyo'):
+                delete_file('%s/%s' % (origdir, filename))
+
+            # Delete empty __init__.py files.
+            origfile = '%s/__init__.py' % origdir
+            if os.path.exists(origfile) and len(open(origfile).read()) == 0:
+                delete_file(origfile)
+
+            # Move sql upgrade scripts.
+            if filename.endswith('.sql'):
+                version, dbms, operation = filename.split('.', 3)[0:3]
+                origfile = '%s/%s' % (origdir, filename)
+                # For instance:  2.postgres.upgrade.sql ->
+                #  002_postgres_upgrade.sql
+                tgtfile = '%s/%03d_%s_%s.sql' % (
+                    versions, int(version), dbms, operation)
+                move_file(origfile, tgtfile)
+
+        # Move Python upgrade script.
+        pyfile = '%s.py' % dirname
+        pyfilepath = '%s/%s' % (origdir, pyfile)
+        if os.path.exists(pyfilepath):
+            tgtfile = '%s/%03d.py' % (versions, int(dirname))
+            move_file(pyfilepath, tgtfile)
+
+        # Try to remove directory. Will fail if it's not empty.
+        delete_directory(origdir)
+
+
+def main():
+    """Main function to be called when using this script."""
+    if len(sys.argv) != 2:
+        usage()
+    migrate_repository(sys.argv[1])
+
+
+if __name__ == '__main__':
+    main()

rhodecode/lib/dbmigrate/migrate/versioning/pathed.py

@@ -0,0 +1,75
+"""
+   A path/directory class.
+"""
+
+import os
+import shutil
+import logging
+
+from rhodecode.lib.dbmigrate.migrate import exceptions
+from rhodecode.lib.dbmigrate.migrate.versioning.config import *
+from rhodecode.lib.dbmigrate.migrate.versioning.util import KeyedInstance
+
+
+log = logging.getLogger(__name__)
+
+class Pathed(KeyedInstance):
+    """
+    A class associated with a path/directory tree.
+
+    Only one instance of this class may exist for a particular file;
+    __new__ will return an existing instance if possible
+    """
+    parent = None
+
+    @classmethod
+    def _key(cls, path):
+        return str(path)
+
+    def __init__(self, path):
+        self.path = path
+        if self.__class__.parent is not None:
+            self._init_parent(path)
+
+    def _init_parent(self, path):
+        """Try to initialize this object's parent, if it has one"""
+        parent_path = self.__class__._parent_path(path)
+        self.parent = self.__class__.parent(parent_path)
+        log.debug("Getting parent %r:%r" % (self.__class__.parent, parent_path))
+        self.parent._init_child(path, self)
+
+    def _init_child(self, child, path):
+        """Run when a child of this object is initialized.
+
+        Parameters: the child object; the path to this object (its
+        parent)
+        """
+
+    @classmethod
+    def _parent_path(cls, path):
+        """
+        Fetch the path of this object's parent from this object's path.
+        """
+        # os.path.dirname(), but strip directories like files (like
+        # unix basename)
+        #
+        # Treat directories like files...
+        if path[-1] == '/':
+            path = path[:-1]
+        ret = os.path.dirname(path)
+        return ret
+
+    @classmethod
+    def require_notfound(cls, path):
+        """Ensures a given path does not already exist"""
+        if os.path.exists(path):
+            raise exceptions.PathFoundError(path)
+
+    @classmethod
+    def require_found(cls, path):
+        """Ensures a given path already exists"""
+        if not os.path.exists(path):
+            raise exceptions.PathNotFoundError(path)
+
+    def __str__(self):
+        return self.path

rhodecode/lib/dbmigrate/migrate/versioning/repository.py

@@ -0,0 +1,231
+"""
+   SQLAlchemy migrate repository management.
+"""
+import os
+import shutil
+import string
+import logging
+
+from pkg_resources import resource_filename
+from tempita import Template as TempitaTemplate
+
+from rhodecode.lib.dbmigrate.migrate import exceptions
+from rhodecode.lib.dbmigrate.migrate.versioning import version, pathed, cfgparse
+from rhodecode.lib.dbmigrate.migrate.versioning.template import Template
+from rhodecode.lib.dbmigrate.migrate.versioning.config import *
+
+
+log = logging.getLogger(__name__)
+
+class Changeset(dict):
+    """A collection of changes to be applied to a database.
+
+    Changesets are bound to a repository and manage a set of
+    scripts from that repository.
+
+    Behaves like a dict, for the most part. Keys are ordered based on step value.
+    """
+
+    def __init__(self, start, *changes, **k):
+        """
+        Give a start version; step must be explicitly stated.
+        """
+        self.step = k.pop('step', 1)
+        self.start = version.VerNum(start)
+        self.end = self.start
+        for change in changes:
+            self.add(change)
+
+    def __iter__(self):
+        return iter(self.items())
+
+    def keys(self):
+        """
+        In a series of upgrades x -> y, keys are version x. Sorted.
+        """
+        ret = super(Changeset, self).keys()
+        # Reverse order if downgrading
+        ret.sort(reverse=(self.step < 1))
+        return ret
+
+    def values(self):
+        return [self[k] for k in self.keys()]
+
+    def items(self):
+        return zip(self.keys(), self.values())
+
+    def add(self, change):
+        """Add new change to changeset"""
+        key = self.end
+        self.end += self.step
+        self[key] = change
+
+    def run(self, *p, **k):
+        """Run the changeset scripts"""
+        for version, script in self:
+            script.run(*p, **k)
+
+
+class Repository(pathed.Pathed):
+    """A project's change script repository"""
+
+    _config = 'migrate.cfg'
+    _versions = 'versions'
+
+    def __init__(self, path):
+        log.debug('Loading repository %s...' % path)
+        self.verify(path)
+        super(Repository, self).__init__(path)
+        self.config = cfgparse.Config(os.path.join(self.path, self._config))
+        self.versions = version.Collection(os.path.join(self.path,
+                                                      self._versions))
+        log.debug('Repository %s loaded successfully' % path)
+        log.debug('Config: %r' % self.config.to_dict())
+
+    @classmethod
+    def verify(cls, path):
+        """
+        Ensure the target path is a valid repository.
+
+        :raises: :exc:`InvalidRepositoryError <migrate.exceptions.InvalidRepositoryError>`
+        """
+        # Ensure the existence of required files
+        try:
+            cls.require_found(path)
+            cls.require_found(os.path.join(path, cls._config))
+            cls.require_found(os.path.join(path, cls._versions))
+        except exceptions.PathNotFoundError, e:
+            raise exceptions.InvalidRepositoryError(path)
+
+    @classmethod
+    def prepare_config(cls, tmpl_dir, name, options=None):
+        """
+        Prepare a project configuration file for a new project.
+
+        :param tmpl_dir: Path to Repository template
+        :param config_file: Name of the config file in Repository template
+        :param name: Repository name
+        :type tmpl_dir: string
+        :type config_file: string
+        :type name: string
+        :returns: Populated config file
+        """
+        if options is None:
+            options = {}
+        options.setdefault('version_table', 'migrate_version')
+        options.setdefault('repository_id', name)
+        options.setdefault('required_dbs', [])
+
+        tmpl = open(os.path.join(tmpl_dir, cls._config)).read()
+        ret = TempitaTemplate(tmpl).substitute(options)
+
+        # cleanup
+        del options['__template_name__']
+
+        return ret
+
+    @classmethod
+    def create(cls, path, name, **opts):
+        """Create a repository at a specified path"""
+        cls.require_notfound(path)
+        theme = opts.pop('templates_theme', None)
+        t_path = opts.pop('templates_path', None)
+
+        # Create repository
+        tmpl_dir = Template(t_path).get_repository(theme=theme)
+        shutil.copytree(tmpl_dir, path)
+
+        # Edit config defaults
+        config_text = cls.prepare_config(tmpl_dir, name, options=opts)
+        fd = open(os.path.join(path, cls._config), 'w')
+        fd.write(config_text)
+        fd.close()
+
+        opts['repository_name'] = name
+
+        # Create a management script
+        manager = os.path.join(path, 'manage.py')
+        Repository.create_manage_file(manager, templates_theme=theme,
+            templates_path=t_path, **opts)
+
+        return cls(path)
+
+    def create_script(self, description, **k):
+        """API to :meth:`migrate.versioning.version.Collection.create_new_python_version`"""
+        self.versions.create_new_python_version(description, **k)
+
+    def create_script_sql(self, database, **k):
+        """API to :meth:`migrate.versioning.version.Collection.create_new_sql_version`"""
+        self.versions.create_new_sql_version(database, **k)
+
+    @property
+    def latest(self):
+        """API to :attr:`migrate.versioning.version.Collection.latest`"""
+        return self.versions.latest
+
+    @property
+    def version_table(self):
+        """Returns version_table name specified in config"""
+        return self.config.get('db_settings', 'version_table')
+
+    @property
+    def id(self):
+        """Returns repository id specified in config"""
+        return self.config.get('db_settings', 'repository_id')
+
+    def version(self, *p, **k):
+        """API to :attr:`migrate.versioning.version.Collection.version`"""
+        return self.versions.version(*p, **k)
+
+    @classmethod
+    def clear(cls):
+        # TODO: deletes repo
+        super(Repository, cls).clear()
+        version.Collection.clear()
+
+    def changeset(self, database, start, end=None):
+        """Create a changeset to migrate this database from ver. start to end/latest.
+
+        :param database: name of database to generate changeset
+        :param start: version to start at
+        :param end: version to end at (latest if None given)
+        :type database: string
+        :type start: int
+        :type end: int
+        :returns: :class:`Changeset instance <migration.versioning.repository.Changeset>`
+        """
+        start = version.VerNum(start)
+
+        if end is None:
+            end = self.latest
+        else:
+            end = version.VerNum(end)
+
+        if start <= end:
+            step = 1
+            range_mod = 1
+            op = 'upgrade'
+        else:
+            step = -1
+            range_mod = 0
+            op = 'downgrade'
+
+        versions = range(start + range_mod, end + range_mod, step)
+        changes = [self.version(v).script(database, op) for v in versions]
+        ret = Changeset(start, step=step, *changes)
+        return ret
+
+    @classmethod
+    def create_manage_file(cls, file_, **opts):
+        """Create a project management script (manage.py)
+        
+        :param file_: Destination file to be written
+        :param opts: Options that are passed to :func:`migrate.versioning.shell.main`
+        """
+        mng_file = Template(opts.pop('templates_path', None))\
+            .get_manage(theme=opts.pop('templates_theme', None))
+
+        tmpl = open(mng_file).read()
+        fd = open(file_, 'w')
+        fd.write(TempitaTemplate(tmpl).substitute(opts))
+        fd.close()

rhodecode/lib/dbmigrate/migrate/versioning/schema.py

@@ -0,0 +1,213
+"""
+   Database schema version management.
+"""
+import sys
+import logging
+
+from sqlalchemy import (Table, Column, MetaData, String, Text, Integer,
+    create_engine)
+from sqlalchemy.sql import and_
+from sqlalchemy import exceptions as sa_exceptions
+from sqlalchemy.sql import bindparam
+
+from rhodecode.lib.dbmigrate.migrate import exceptions
+from rhodecode.lib.dbmigrate.migrate.versioning import genmodel, schemadiff
+from rhodecode.lib.dbmigrate.migrate.versioning.repository import Repository
+from rhodecode.lib.dbmigrate.migrate.versioning.util import load_model
+from rhodecode.lib.dbmigrate.migrate.versioning.version import VerNum
+
+
+log = logging.getLogger(__name__)
+
+class ControlledSchema(object):
+    """A database under version control"""
+
+    def __init__(self, engine, repository):
+        if isinstance(repository, basestring):
+            repository = Repository(repository)
+        self.engine = engine
+        self.repository = repository
+        self.meta = MetaData(engine)
+        self.load()
+
+    def __eq__(self, other):
+        """Compare two schemas by repositories and versions"""
+        return (self.repository is other.repository \
+            and self.version == other.version)
+
+    def load(self):
+        """Load controlled schema version info from DB"""
+        tname = self.repository.version_table
+        try:
+            if not hasattr(self, 'table') or self.table is None:
+                    self.table = Table(tname, self.meta, autoload=True)
+
+            result = self.engine.execute(self.table.select(
+                self.table.c.repository_id == str(self.repository.id)))
+
+            data = list(result)[0]
+        except:
+            cls, exc, tb = sys.exc_info()
+            raise exceptions.DatabaseNotControlledError, exc.__str__(), tb
+
+        self.version = data['version']
+        return data
+
+    def drop(self):
+        """
+        Remove version control from a database.
+        """
+        try:
+            self.table.drop()
+        except (sa_exceptions.SQLError):
+            raise exceptions.DatabaseNotControlledError(str(self.table))
+
+    def changeset(self, version=None):
+        """API to Changeset creation.
+        
+        Uses self.version for start version and engine.name
+        to get database name.
+        """
+        database = self.engine.name
+        start_ver = self.version
+        changeset = self.repository.changeset(database, start_ver, version)
+        return changeset
+
+    def runchange(self, ver, change, step):
+        startver = ver
+        endver = ver + step
+        # Current database version must be correct! Don't run if corrupt!
+        if self.version != startver:
+            raise exceptions.InvalidVersionError("%s is not %s" % \
+                                                     (self.version, startver))
+        # Run the change
+        change.run(self.engine, step)
+
+        # Update/refresh database version
+        self.update_repository_table(startver, endver)
+        self.load()
+
+    def update_repository_table(self, startver, endver):
+        """Update version_table with new information"""
+        update = self.table.update(and_(self.table.c.version == int(startver),
+             self.table.c.repository_id == str(self.repository.id)))
+        self.engine.execute(update, version=int(endver))
+
+    def upgrade(self, version=None):
+        """
+        Upgrade (or downgrade) to a specified version, or latest version.
+        """
+        changeset = self.changeset(version)
+        for ver, change in changeset:
+            self.runchange(ver, change, changeset.step)
+
+    def update_db_from_model(self, model):
+        """
+        Modify the database to match the structure of the current Python model.
+        """
+        model = load_model(model)
+
+        diff = schemadiff.getDiffOfModelAgainstDatabase(
+            model, self.engine, excludeTables=[self.repository.version_table]
+            )
+        genmodel.ModelGenerator(diff,self.engine).applyModel()
+
+        self.update_repository_table(self.version, int(self.repository.latest))
+
+        self.load()
+
+    @classmethod
+    def create(cls, engine, repository, version=None):
+        """
+        Declare a database to be under a repository's version control.
+
+        :raises: :exc:`DatabaseAlreadyControlledError`
+        :returns: :class:`ControlledSchema`
+        """
+        # Confirm that the version # is valid: positive, integer,
+        # exists in repos
+        if isinstance(repository, basestring):
+            repository = Repository(repository)
+        version = cls._validate_version(repository, version)
+        table = cls._create_table_version(engine, repository, version)
+        # TODO: history table
+        # Load repository information and return
+        return cls(engine, repository)
+
+    @classmethod
+    def _validate_version(cls, repository, version):
+        """
+        Ensures this is a valid version number for this repository.
+
+        :raises: :exc:`InvalidVersionError` if invalid
+        :return: valid version number
+        """
+        if version is None:
+            version = 0
+        try:
+            version = VerNum(version) # raises valueerror
+            if version < 0 or version > repository.latest:
+                raise ValueError()
+        except ValueError:
+            raise exceptions.InvalidVersionError(version)
+        return version
+
+    @classmethod
+    def _create_table_version(cls, engine, repository, version):
+        """
+        Creates the versioning table in a database.
+
+        :raises: :exc:`DatabaseAlreadyControlledError`
+        """
+        # Create tables
+        tname = repository.version_table
+        meta = MetaData(engine)
+
+        table = Table(
+            tname, meta,
+            Column('repository_id', String(250), primary_key=True),
+            Column('repository_path', Text),
+            Column('version', Integer), )
+
+        # there can be multiple repositories/schemas in the same db
+        if not table.exists():
+            table.create()
+
+        # test for existing repository_id
+        s = table.select(table.c.repository_id == bindparam("repository_id"))
+        result = engine.execute(s, repository_id=repository.id)
+        if result.fetchone():
+            raise exceptions.DatabaseAlreadyControlledError
+
+        # Insert data
+        engine.execute(table.insert().values(
+                           repository_id=repository.id,
+                           repository_path=repository.path,
+                           version=int(version)))
+        return table
+
+    @classmethod
+    def compare_model_to_db(cls, engine, model, repository):
+        """
+        Compare the current model against the current database.
+        """
+        if isinstance(repository, basestring):
+            repository = Repository(repository)
+        model = load_model(model)
+
+        diff = schemadiff.getDiffOfModelAgainstDatabase(
+            model, engine, excludeTables=[repository.version_table])
+        return diff
+
+    @classmethod
+    def create_model(cls, engine, repository, declarative=False):
+        """
+        Dump the current database as a Python model.
+        """
+        if isinstance(repository, basestring):
+            repository = Repository(repository)
+
+        diff = schemadiff.getDiffOfModelAgainstDatabase(
+            MetaData(), engine, excludeTables=[repository.version_table]
+            )
+        return genmodel.ModelGenerator(diff, engine, declarative).toPython()

rhodecode/lib/dbmigrate/migrate/versioning/schemadiff.py

@@ -0,0 +1,285
+"""
+   Schema differencing support.
+"""
+
+import logging
+import sqlalchemy
+
+from rhodecode.lib.dbmigrate.migrate.changeset import SQLA_06
+from sqlalchemy.types import Float
+
+log = logging.getLogger(__name__)
+
+def getDiffOfModelAgainstDatabase(metadata, engine, excludeTables=None):
+    """
+    Return differences of model against database.
+
+    :return: object which will evaluate to :keyword:`True` if there \
+      are differences else :keyword:`False`.
+    """
+    return SchemaDiff(metadata,
+                      sqlalchemy.MetaData(engine, reflect=True),
+                      labelA='model',
+                      labelB='database',
+                      excludeTables=excludeTables)
+
+
+def getDiffOfModelAgainstModel(metadataA, metadataB, excludeTables=None):
+    """
+    Return differences of model against another model.
+
+    :return: object which will evaluate to :keyword:`True` if there \
+      are differences else :keyword:`False`.
+    """
+    return SchemaDiff(metadataA, metadataB, excludeTables)
+
+
+class ColDiff(object):
+    """
+    Container for differences in one :class:`~sqlalchemy.schema.Column`
+    between two :class:`~sqlalchemy.schema.Table` instances, ``A``
+    and ``B``.
+    
+    .. attribute:: col_A
+
+      The :class:`~sqlalchemy.schema.Column` object for A.
+      
+    .. attribute:: col_B
+
+      The :class:`~sqlalchemy.schema.Column` object for B.
+
+    .. attribute:: type_A
+
+      The most generic type of the :class:`~sqlalchemy.schema.Column`
+      object in A. 
+      
+    .. attribute:: type_B
+
+      The most generic type of the :class:`~sqlalchemy.schema.Column`
+      object in A. 
+      
+    """
+    
+    diff = False
+
+    def __init__(self,col_A,col_B):
+        self.col_A = col_A
+        self.col_B = col_B
+
+        self.type_A = col_A.type
+        self.type_B = col_B.type
+
+        self.affinity_A = self.type_A._type_affinity
+        self.affinity_B = self.type_B._type_affinity
+
+        if self.affinity_A is not self.affinity_B:
+            self.diff = True
+            return
+
+        if isinstance(self.type_A,Float) or isinstance(self.type_B,Float):
+            if not (isinstance(self.type_A,Float) and isinstance(self.type_B,Float)):
+                self.diff=True
+                return
+
+        for attr in ('precision','scale','length'):
+            A = getattr(self.type_A,attr,None)
+            B = getattr(self.type_B,attr,None)
+            if not (A is None or B is None) and A!=B:
+                self.diff=True
+                return
+        
+    def __nonzero__(self):
+        return self.diff
+    
+class TableDiff(object):
+    """
+    Container for differences in one :class:`~sqlalchemy.schema.Table`
+    between two :class:`~sqlalchemy.schema.MetaData` instances, ``A``
+    and ``B``.
+
+    .. attribute:: columns_missing_from_A
+
+      A sequence of column names that were found in B but weren't in
+      A.
+      
+    .. attribute:: columns_missing_from_B
+
+      A sequence of column names that were found in A but weren't in
+      B.
+      
+    .. attribute:: columns_different
+
+      A dictionary containing information about columns that were
+      found to be different.
+      It maps column names to a :class:`ColDiff` objects describing the
+      differences found.
+    """
+    __slots__ = (
+        'columns_missing_from_A',
+        'columns_missing_from_B',
+        'columns_different',
+        )
+
+    def __nonzero__(self):
+        return bool(
+            self.columns_missing_from_A or
+            self.columns_missing_from_B or
+            self.columns_different
+            )
+    
+class SchemaDiff(object):
+    """
+    Compute the difference between two :class:`~sqlalchemy.schema.MetaData`
+    objects.
+
+    The string representation of a :class:`SchemaDiff` will summarise
+    the changes found between the two
+    :class:`~sqlalchemy.schema.MetaData` objects.
+
+    The length of a :class:`SchemaDiff` will give the number of
+    changes found, enabling it to be used much like a boolean in
+    expressions.
+        
+    :param metadataA:
+      First :class:`~sqlalchemy.schema.MetaData` to compare.
+      
+    :param metadataB:
+      Second :class:`~sqlalchemy.schema.MetaData` to compare.
+      
+    :param labelA:
+      The label to use in messages about the first
+      :class:`~sqlalchemy.schema.MetaData`. 
+    
+    :param labelB: 
+      The label to use in messages about the second
+      :class:`~sqlalchemy.schema.MetaData`. 
+    
+    :param excludeTables:
+      A sequence of table names to exclude.
+      
+    .. attribute:: tables_missing_from_A
+
+      A sequence of table names that were found in B but weren't in
+      A.
+      
+    .. attribute:: tables_missing_from_B
+
+      A sequence of table names that were found in A but weren't in
+      B.
+      
+    .. attribute:: tables_different
+
+      A dictionary containing information about tables that were found
+      to be different.
+      It maps table names to a :class:`TableDiff` objects describing the
+      differences found.
+    """
+
+    def __init__(self,
+                 metadataA, metadataB,
+                 labelA='metadataA',
+                 labelB='metadataB',
+                 excludeTables=None):
+
+        self.metadataA, self.metadataB = metadataA, metadataB
+        self.labelA, self.labelB = labelA, labelB
+        self.label_width = max(len(labelA),len(labelB))
+        excludeTables = set(excludeTables or [])
+
+        A_table_names = set(metadataA.tables.keys())
+        B_table_names = set(metadataB.tables.keys())
+
+        self.tables_missing_from_A = sorted(
+            B_table_names - A_table_names - excludeTables
+            )
+        self.tables_missing_from_B = sorted(
+            A_table_names - B_table_names - excludeTables
+            )
+        
+        self.tables_different = {}
+        for table_name in A_table_names.intersection(B_table_names):
+
+            td = TableDiff()
+            
+            A_table = metadataA.tables[table_name]
+            B_table = metadataB.tables[table_name]
+            
+            A_column_names = set(A_table.columns.keys())
+            B_column_names = set(B_table.columns.keys())
+
+            td.columns_missing_from_A = sorted(
+                B_column_names - A_column_names
+                )
+            
+            td.columns_missing_from_B = sorted(
+                A_column_names - B_column_names
+                )
+            
+            td.columns_different = {}
+
+            for col_name in A_column_names.intersection(B_column_names):
+
+                cd = ColDiff(
+                    A_table.columns.get(col_name),
+                    B_table.columns.get(col_name)
+                    )
+
+                if cd:
+                    td.columns_different[col_name]=cd
+                
+            # XXX - index and constraint differences should
+            #       be checked for here
+
+            if td:
+                self.tables_different[table_name]=td
+
+    def __str__(self):
+        ''' Summarize differences. '''
+        out = []
+        column_template ='      %%%is: %%r' % self.label_width
+        
+        for names,label in (
+            (self.tables_missing_from_A,self.labelA),
+            (self.tables_missing_from_B,self.labelB),
+            ):
+            if names:
+                out.append(
+                    '  tables missing from %s: %s' % (
+                        label,', '.join(sorted(names))
+                        )
+                    )
+                
+        for name,td in sorted(self.tables_different.items()):
+            out.append(
+               '  table with differences: %s' % name
+               )
+            for names,label in (
+                (td.columns_missing_from_A,self.labelA),
+                (td.columns_missing_from_B,self.labelB),
+                ):
+                if names:
+                    out.append(
+                        '    %s missing these columns: %s' % (
+                            label,', '.join(sorted(names))
+                            )
+                        )
+            for name,cd in td.columns_different.items():
+                out.append('    column with differences: %s' % name)
+                out.append(column_template % (self.labelA,cd.col_A))
+                out.append(column_template % (self.labelB,cd.col_B))
+                
+        if out:
+            out.insert(0, 'Schema diffs:')
+            return '\n'.join(out)
+        else:
+            return 'No schema diffs'
+
+    def __len__(self):
+        """
+        Used in bool evaluation, return of 0 means no diffs.
+        """
+        return (
+            len(self.tables_missing_from_A) +
+            len(self.tables_missing_from_B) +
+            len(self.tables_different)
+            )

rhodecode/lib/dbmigrate/migrate/versioning/script/__init__.py

@@ -0,0 +1,6
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from rhodecode.lib.dbmigrate.migrate.versioning.script.base import BaseScript
+from rhodecode.lib.dbmigrate.migrate.versioning.script.py import PythonScript
+from rhodecode.lib.dbmigrate.migrate.versioning.script.sql import SqlScript

rhodecode/lib/dbmigrate/migrate/versioning/script/base.py

@@ -0,0 +1,57
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import logging
+
+from rhodecode.lib.dbmigrate.migrate import exceptions
+from rhodecode.lib.dbmigrate.migrate.versioning.config import operations
+from rhodecode.lib.dbmigrate.migrate.versioning import pathed
+
+
+log = logging.getLogger(__name__)
+
+class BaseScript(pathed.Pathed):
+    """Base class for other types of scripts.
+    All scripts have the following properties:
+
+    source (script.source())
+      The source code of the script
+    version (script.version())
+      The version number of the script
+    operations (script.operations())
+      The operations defined by the script: upgrade(), downgrade() or both.
+      Returns a tuple of operations.
+      Can also check for an operation with ex. script.operation(Script.ops.up)
+    """ # TODO: sphinxfy this and implement it correctly
+
+    def __init__(self, path):
+        log.debug('Loading script %s...' % path)
+        self.verify(path)
+        super(BaseScript, self).__init__(path)
+        log.debug('Script %s loaded successfully' % path)
+    
+    @classmethod
+    def verify(cls, path):
+        """Ensure this is a valid script
+        This version simply ensures the script file's existence
+
+        :raises: :exc:`InvalidScriptError <migrate.exceptions.InvalidScriptError>`
+        """
+        try:
+            cls.require_found(path)
+        except:
+            raise exceptions.InvalidScriptError(path)
+
+    def source(self):
+        """:returns: source code of the script.
+        :rtype: string
+        """
+        fd = open(self.path)
+        ret = fd.read()
+        fd.close()
+        return ret
+
+    def run(self, engine):
+        """Core of each BaseScript subclass.
+        This method executes the script.
+        """
+        raise NotImplementedError()

rhodecode/lib/dbmigrate/migrate/versioning/script/py.py

@@ -0,0 +1,159
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import shutil
+import warnings
+import logging
+from StringIO import StringIO
+
+from rhodecode.lib.dbmigrate import migrate
+from rhodecode.lib.dbmigrate.migrate.versioning import genmodel, schemadiff
+from rhodecode.lib.dbmigrate.migrate.versioning.config import operations
+from rhodecode.lib.dbmigrate.migrate.versioning.template import Template
+from rhodecode.lib.dbmigrate.migrate.versioning.script import base
+from rhodecode.lib.dbmigrate.migrate.versioning.util import import_path, load_model, with_engine
+from rhodecode.lib.dbmigrate.migrate.exceptions import MigrateDeprecationWarning, InvalidScriptError, ScriptError
+
+log = logging.getLogger(__name__)
+__all__ = ['PythonScript']
+
+
+class PythonScript(base.BaseScript):
+    """Base for Python scripts"""
+
+    @classmethod
+    def create(cls, path, **opts):
+        """Create an empty migration script at specified path
+        
+        :returns: :class:`PythonScript instance <migrate.versioning.script.py.PythonScript>`"""
+        cls.require_notfound(path)
+
+        src = Template(opts.pop('templates_path', None)).get_script(theme=opts.pop('templates_theme', None))
+        shutil.copy(src, path)
+
+        return cls(path)
+
+    @classmethod
+    def make_update_script_for_model(cls, engine, oldmodel,
+                                     model, repository, **opts):
+        """Create a migration script based on difference between two SA models.
+        
+        :param repository: path to migrate repository
+        :param oldmodel: dotted.module.name:SAClass or SAClass object
+        :param model: dotted.module.name:SAClass or SAClass object
+        :param engine: SQLAlchemy engine
+        :type repository: string or :class:`Repository instance <migrate.versioning.repository.Repository>`
+        :type oldmodel: string or Class
+        :type model: string or Class
+        :type engine: Engine instance
+        :returns: Upgrade / Downgrade script
+        :rtype: string
+        """
+
+        if isinstance(repository, basestring):
+            # oh dear, an import cycle!
+            from rhodecode.lib.dbmigrate.migrate.versioning.repository import Repository
+            repository = Repository(repository)
+
+        oldmodel = load_model(oldmodel)
+        model = load_model(model)
+
+        # Compute differences.
+        diff = schemadiff.getDiffOfModelAgainstModel(
+            oldmodel,
+            model,
+            excludeTables=[repository.version_table])
+        # TODO: diff can be False (there is no difference?)
+        decls, upgradeCommands, downgradeCommands = \
+            genmodel.ModelGenerator(diff, engine).toUpgradeDowngradePython()
+
+        # Store differences into file.
+        src = Template(opts.pop('templates_path', None)).get_script(opts.pop('templates_theme', None))
+        f = open(src)
+        contents = f.read()
+        f.close()
+
+        # generate source
+        search = 'def upgrade(migrate_engine):'
+        contents = contents.replace(search, '\n\n'.join((decls, search)), 1)
+        if upgradeCommands:
+            contents = contents.replace('    pass', upgradeCommands, 1)
+        if downgradeCommands:
+            contents = contents.replace('    pass', downgradeCommands, 1)
+        return contents
+
+    @classmethod
+    def verify_module(cls, path):
+        """Ensure path is a valid script
+        
+        :param path: Script location
+        :type path: string
+        :raises: :exc:`InvalidScriptError <migrate.exceptions.InvalidScriptError>`
+        :returns: Python module
+        """
+        # Try to import and get the upgrade() func
+        module = import_path(path)
+        try:
+            assert callable(module.upgrade)
+        except Exception, e:
+            raise InvalidScriptError(path + ': %s' % str(e))
+        return module
+
+    def preview_sql(self, url, step, **args):
+        """Mocks SQLAlchemy Engine to store all executed calls in a string 
+        and runs :meth:`PythonScript.run <migrate.versioning.script.py.PythonScript.run>`
+
+        :returns: SQL file
+        """
+        buf = StringIO()
+        args['engine_arg_strategy'] = 'mock'
+        args['engine_arg_executor'] = lambda s, p = '': buf.write(str(s) + p)
+
+        @with_engine
+        def go(url, step, **kw):
+            engine = kw.pop('engine')
+            self.run(engine, step)
+            return buf.getvalue()
+
+        return go(url, step, **args)
+
+    def run(self, engine, step):
+        """Core method of Script file. 
+        Exectues :func:`update` or :func:`downgrade` functions
+
+        :param engine: SQLAlchemy Engine
+        :param step: Operation to run
+        :type engine: string
+        :type step: int
+        """
+        if step > 0:
+            op = 'upgrade'
+        elif step < 0:
+            op = 'downgrade'
+        else:
+            raise ScriptError("%d is not a valid step" % step)
+
+        funcname = base.operations[op]
+        script_func = self._func(funcname)
+
+        try:
+            script_func(engine)
+        except TypeError:
+            warnings.warn("upgrade/downgrade functions must accept engine"
+                " parameter (since version > 0.5.4)", MigrateDeprecationWarning)
+            raise
+
+    @property
+    def module(self):
+        """Calls :meth:`migrate.versioning.script.py.verify_module`
+        and returns it.
+        """
+        if not hasattr(self, '_module'):
+            self._module = self.verify_module(self.path)
+        return self._module
+
+    def _func(self, funcname):
+        if not hasattr(self.module, funcname):
+            msg = "Function '%s' is not defined in this script"
+            raise ScriptError(msg % funcname)
+        return getattr(self.module, funcname)

rhodecode/lib/dbmigrate/migrate/versioning/script/sql.py

@@ -0,0 +1,48
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import logging
+import shutil
+
+from rhodecode.lib.dbmigrate.migrate.versioning.script import base
+from rhodecode.lib.dbmigrate.migrate.versioning.template import Template
+
+
+log = logging.getLogger(__name__)
+
+class SqlScript(base.BaseScript):
+    """A file containing plain SQL statements."""
+
+    @classmethod
+    def create(cls, path, **opts):
+        """Create an empty migration script at specified path
+        
+        :returns: :class:`SqlScript instance <migrate.versioning.script.sql.SqlScript>`"""
+        cls.require_notfound(path)
+        src = Template(opts.pop('templates_path', None)).get_sql_script(theme=opts.pop('templates_theme', None))
+        shutil.copy(src, path)
+        return cls(path)
+
+    # TODO: why is step parameter even here?
+    def run(self, engine, step=None, executemany=True):
+        """Runs SQL script through raw dbapi execute call"""
+        text = self.source()
+        # Don't rely on SA's autocommit here
+        # (SA uses .startswith to check if a commit is needed. What if script
+        # starts with a comment?)
+        conn = engine.connect()
+        try:
+            trans = conn.begin()
+            try:
+                # HACK: SQLite doesn't allow multiple statements through
+                # its execute() method, but it provides executescript() instead
+                dbapi = conn.engine.raw_connection()
+                if executemany and getattr(dbapi, 'executescript', None):
+                    dbapi.executescript(text)
+                else:
+                    conn.execute(text)
+                trans.commit()
+            except:
+                trans.rollback()
+                raise
+        finally:
+            conn.close()

rhodecode/lib/dbmigrate/migrate/versioning/shell.py

@@ -0,0 +1,215
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""The migrate command-line tool."""
+
+import sys
+import inspect
+import logging
+from optparse import OptionParser, BadOptionError
+
+from rhodecode.lib.dbmigrate.migrate import exceptions
+from rhodecode.lib.dbmigrate.migrate.versioning import api
+from rhodecode.lib.dbmigrate.migrate.versioning.config import *
+from rhodecode.lib.dbmigrate.migrate.versioning.util import asbool
+
+
+alias = dict(
+    s=api.script,
+    vc=api.version_control,
+    dbv=api.db_version,
+    v=api.version,
+)
+
+def alias_setup():
+    global alias
+    for key, val in alias.iteritems():
+        setattr(api, key, val)
+alias_setup()
+
+
+class PassiveOptionParser(OptionParser):
+
+    def _process_args(self, largs, rargs, values):
+        """little hack to support all --some_option=value parameters"""
+
+        while rargs:
+            arg = rargs[0]
+            if arg == "--":
+                del rargs[0]
+                return
+            elif arg[0:2] == "--":
+                # if parser does not know about the option
+                # pass it along (make it anonymous)
+                try:
+                    opt = arg.split('=', 1)[0]
+                    self._match_long_opt(opt)
+                except BadOptionError:
+                    largs.append(arg)
+                    del rargs[0]
+                else:
+                    self._process_long_opt(rargs, values)
+            elif arg[:1] == "-" and len(arg) > 1:
+                self._process_short_opts(rargs, values)
+            elif self.allow_interspersed_args:
+                largs.append(arg)
+                del rargs[0]
+
+def main(argv=None, **kwargs):
+    """Shell interface to :mod:`migrate.versioning.api`.
+
+    kwargs are default options that can be overriden with passing
+    --some_option as command line option
+
+    :param disable_logging: Let migrate configure logging
+    :type disable_logging: bool
+    """
+    if argv is not None:
+        argv = argv
+    else:
+        argv = list(sys.argv[1:])
+    commands = list(api.__all__)
+    commands.sort()
+
+    usage = """%%prog COMMAND ...
+
+    Available commands:
+        %s
+
+    Enter "%%prog help COMMAND" for information on a particular command.
+    """ % '\n\t'.join(["%s - %s" % (command.ljust(28),
+                    api.command_desc.get(command)) for command in commands])
+
+    parser = PassiveOptionParser(usage=usage)
+    parser.add_option("-d", "--debug",
+                     action="store_true",
+                     dest="debug",
+                     default=False,
+                     help="Shortcut to turn on DEBUG mode for logging")
+    parser.add_option("-q", "--disable_logging",
+                      action="store_true",
+                      dest="disable_logging",
+                      default=False,
+                      help="Use this option to disable logging configuration")
+    help_commands = ['help', '-h', '--help']
+    HELP = False
+
+    try:
+        command = argv.pop(0)
+        if command in help_commands:
+            HELP = True
+            command = argv.pop(0)
+    except IndexError:
+        parser.print_help()
+        return
+
+    command_func = getattr(api, command, None)
+    if command_func is None or command.startswith('_'):
+        parser.error("Invalid command %s" % command)
+
+    parser.set_usage(inspect.getdoc(command_func))
+    f_args, f_varargs, f_kwargs, f_defaults = inspect.getargspec(command_func)
+    for arg in f_args:
+        parser.add_option(
+            "--%s" % arg,
+            dest=arg,
+            action='store',
+            type="string")
+
+    # display help of the current command
+    if HELP:
+        parser.print_help()
+        return
+
+    options, args = parser.parse_args(argv)
+
+    # override kwargs with anonymous parameters
+    override_kwargs = dict()
+    for arg in list(args):
+        if arg.startswith('--'):
+            args.remove(arg)
+            if '=' in arg:
+                opt, value = arg[2:].split('=', 1)
+            else:
+                opt = arg[2:]
+                value = True
+            override_kwargs[opt] = value
+
+    # override kwargs with options if user is overwriting
+    for key, value in options.__dict__.iteritems():
+        if value is not None:
+            override_kwargs[key] = value
+
+    # arguments that function accepts without passed kwargs
+    f_required = list(f_args)
+    candidates = dict(kwargs)
+    candidates.update(override_kwargs)
+    for key, value in candidates.iteritems():
+        if key in f_args:
+            f_required.remove(key)
+
+    # map function arguments to parsed arguments
+    for arg in args:
+        try:
+            kw = f_required.pop(0)
+        except IndexError:
+            parser.error("Too many arguments for command %s: %s" % (command,
+                                                                    arg))
+        kwargs[kw] = arg
+
+    # apply overrides
+    kwargs.update(override_kwargs)
+
+    # configure options
+    for key, value in options.__dict__.iteritems():
+        kwargs.setdefault(key, value)
+
+    # configure logging
+    if not asbool(kwargs.pop('disable_logging', False)):
+        # filter to log =< INFO into stdout and rest to stderr
+        class SingleLevelFilter(logging.Filter):
+            def __init__(self, min=None, max=None):
+                self.min = min or 0
+                self.max = max or 100
+
+            def filter(self, record):
+                return self.min <= record.levelno <= self.max
+
+        logger = logging.getLogger()
+        h1 = logging.StreamHandler(sys.stdout)
+        f1 = SingleLevelFilter(max=logging.INFO)
+        h1.addFilter(f1)
+        h2 = logging.StreamHandler(sys.stderr)
+        f2 = SingleLevelFilter(min=logging.WARN)
+        h2.addFilter(f2)
+        logger.addHandler(h1)
+        logger.addHandler(h2)
+
+        if options.debug:
+            logger.setLevel(logging.DEBUG)
+        else:
+            logger.setLevel(logging.INFO)
+
+    log = logging.getLogger(__name__)
+
+    # check if all args are given
+    try:
+        num_defaults = len(f_defaults)
+    except TypeError:
+        num_defaults = 0
+    f_args_default = f_args[len(f_args) - num_defaults:]
+    required = list(set(f_required) - set(f_args_default))
+    if required:
+        parser.error("Not enough arguments for command %s: %s not specified" \
+            % (command, ', '.join(required)))
+
+    # handle command
+    try:
+        ret = command_func(**kwargs)
+        if ret is not None:
+            log.info(ret)
+    except (exceptions.UsageError, exceptions.KnownError), e:
+        parser.error(e.args[0])
+
+if __name__ == "__main__":
+    main()

rhodecode/lib/dbmigrate/migrate/versioning/template.py

@@ -0,0 +1,94
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import shutil
+import sys
+
+from pkg_resources import resource_filename
+
+from rhodecode.lib.dbmigrate.migrate.versioning.config import *
+from rhodecode.lib.dbmigrate.migrate.versioning import pathed
+
+
+class Collection(pathed.Pathed):
+    """A collection of templates of a specific type"""
+    _mask = None
+
+    def get_path(self, file):
+        return os.path.join(self.path, str(file))
+
+
+class RepositoryCollection(Collection):
+    _mask = '%s'
+
+class ScriptCollection(Collection):
+    _mask = '%s.py_tmpl'
+
+class ManageCollection(Collection):
+    _mask = '%s.py_tmpl'
+
+class SQLScriptCollection(Collection):
+    _mask = '%s.py_tmpl'
+
+class Template(pathed.Pathed):
+    """Finds the paths/packages of various Migrate templates.
+    
+    :param path: Templates are loaded from rhodecode.lib.dbmigrate.migrate package
+    if `path` is not provided.
+    """
+    pkg = 'rhodecode.lib.dbmigrate.migrate.versioning.templates'
+    _manage = 'manage.py_tmpl'
+
+    def __new__(cls, path=None):
+        if path is None:
+            path = cls._find_path(cls.pkg)
+        return super(Template, cls).__new__(cls, path)
+
+    def __init__(self, path=None):
+        if path is None:
+            path = Template._find_path(self.pkg)
+        super(Template, self).__init__(path)
+        self.repository = RepositoryCollection(os.path.join(path, 'repository'))
+        self.script = ScriptCollection(os.path.join(path, 'script'))
+        self.manage = ManageCollection(os.path.join(path, 'manage'))
+        self.sql_script = SQLScriptCollection(os.path.join(path, 'sql_script'))
+
+    @classmethod
+    def _find_path(cls, pkg):
+        """Returns absolute path to dotted python package."""
+        tmp_pkg = pkg.rsplit('.', 1)
+
+        if len(tmp_pkg) != 1:
+            return resource_filename(tmp_pkg[0], tmp_pkg[1])
+        else:
+            return resource_filename(tmp_pkg[0], '')
+
+    def _get_item(self, collection, theme=None):
+        """Locates and returns collection.
+        
+        :param collection: name of collection to locate
+        :param type_: type of subfolder in collection (defaults to "_default")
+        :returns: (package, source)
+        :rtype: str, str
+        """
+        item = getattr(self, collection)
+        theme_mask = getattr(item, '_mask')
+        theme = theme_mask % (theme or 'default')
+        return item.get_path(theme)
+
+    def get_repository(self, *a, **kw):
+        """Calls self._get_item('repository', *a, **kw)"""
+        return self._get_item('repository', *a, **kw)
+
+    def get_script(self, *a, **kw):
+        """Calls self._get_item('script', *a, **kw)"""
+        return self._get_item('script', *a, **kw)
+
+    def get_sql_script(self, *a, **kw):
+        """Calls self._get_item('sql_script', *a, **kw)"""
+        return self._get_item('sql_script', *a, **kw)
+
+    def get_manage(self, *a, **kw):
+        """Calls self._get_item('manage', *a, **kw)"""
+        return self._get_item('manage', *a, **kw)

rhodecode/lib/dbmigrate/migrate/versioning/templates/manage.py_tmpl

@@ -0,0 +1,5
+#!/usr/bin/env python
+from migrate.versioning.shell import main
+
+if __name__ == '__main__':
+    main(%(defaults)s)

rhodecode/lib/dbmigrate/migrate/versioning/templates/manage/default.py_tmpl

@@ -0,0 +1,10
+#!/usr/bin/env python
+from migrate.versioning.shell import main
+
+{{py:
+_vars = locals().copy()
+del _vars['__template_name__']
+_vars.pop('repository_name', None)
+defaults = ", ".join(["%s='%s'" % var for var in _vars.iteritems()])
+}}
+main({{ defaults }})

rhodecode/lib/dbmigrate/migrate/versioning/templates/manage/pylons.py_tmpl

@@ -0,0 +1,29
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+import sys
+
+from sqlalchemy import engine_from_config
+from paste.deploy.loadwsgi import ConfigLoader
+
+from migrate.versioning.shell import main
+from {{ locals().pop('repository_name') }}.model import migrations
+
+
+if '-c' in sys.argv:
+    pos = sys.argv.index('-c')
+    conf_path = sys.argv[pos + 1]
+    del sys.argv[pos:pos + 2]
+else:
+    conf_path = 'development.ini'
+
+{{py:
+_vars = locals().copy()
+del _vars['__template_name__']
+defaults = ", ".join(["%s='%s'" % var for var in _vars.iteritems()])
+}}
+
+conf_dict = ConfigLoader(conf_path).parser._sections['app:main']
+
+# migrate supports passing url as an existing Engine instance (since 0.6.0)
+# usage: migrate -c path/to/config.ini COMMANDS
+main(url=engine_from_config(conf_dict), repository=migrations.__path__[0],{{ defaults }})

rhodecode/lib/dbmigrate/migrate/versioning/templates/repository/default/README

@@ -0,0 +1,4
+This is a database migration repository.
+
+More information at
+http://code.google.com/p/sqlalchemy-migrate/

rhodecode/lib/dbmigrate/migrate/versioning/templates/repository/default/migrate.cfg

@@ -0,0 +1,20
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id={{ locals().pop('repository_id') }}
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to 
+# change the table name in each database too. 
+version_table={{ locals().pop('version_table') }}
+
+# When committing a change script, Migrate will attempt to generate the 
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the 
+# commit continues, perhaps ending successfully. 
+# Databases in this list MUST compile successfully during a commit, or the 
+# entire commit will fail. List the databases your application will actually 
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs={{ locals().pop('required_dbs') }}

rhodecode/lib/dbmigrate/migrate/versioning/templates/repository/pylons/README

@@ -0,0 +1,4
+This is a database migration repository.
+
+More information at
+http://code.google.com/p/sqlalchemy-migrate/

rhodecode/lib/dbmigrate/migrate/versioning/templates/repository/pylons/migrate.cfg

@@ -0,0 +1,20
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id={{ locals().pop('repository_id') }}
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to 
+# change the table name in each database too. 
+version_table={{ locals().pop('version_table') }}
+
+# When committing a change script, Migrate will attempt to generate the 
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the 
+# commit continues, perhaps ending successfully. 
+# Databases in this list MUST compile successfully during a commit, or the 
+# entire commit will fail. List the databases your application will actually 
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs={{ locals().pop('required_dbs') }}

rhodecode/lib/dbmigrate/migrate/versioning/templates/script/default.py_tmpl

@@ -0,0 +1,11
+from sqlalchemy import *
+from migrate import *
+
+def upgrade(migrate_engine):
+    # Upgrade operations go here. Don't create your own engine; bind migrate_engine
+    # to your metadata
+    pass
+
+def downgrade(migrate_engine):
+    # Operations to reverse the above upgrade go here.
+    pass

rhodecode/lib/dbmigrate/migrate/versioning/templates/script/pylons.py_tmpl

@@ -0,0 +1,11
+from sqlalchemy import *
+from migrate import *
+
+def upgrade(migrate_engine):
+    # Upgrade operations go here. Don't create your own engine; bind migrate_engine
+    # to your metadata
+    pass
+
+def downgrade(migrate_engine):
+    # Operations to reverse the above upgrade go here.
+    pass

rhodecode/lib/dbmigrate/migrate/versioning/util/__init__.py

@@ -0,0 +1,179
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+""".. currentmodule:: migrate.versioning.util"""
+
+import warnings
+import logging
+from decorator import decorator
+from pkg_resources import EntryPoint
+
+from sqlalchemy import create_engine
+from sqlalchemy.engine import Engine
+from sqlalchemy.pool import StaticPool
+
+from rhodecode.lib.dbmigrate.migrate import exceptions
+from rhodecode.lib.dbmigrate.migrate.versioning.util.keyedinstance import KeyedInstance
+from rhodecode.lib.dbmigrate.migrate.versioning.util.importpath import import_path
+
+
+log = logging.getLogger(__name__)
+
+def load_model(dotted_name):
+    """Import module and use module-level variable".
+
+    :param dotted_name: path to model in form of string: ``some.python.module:Class``
+    
+    .. versionchanged:: 0.5.4
+
+    """
+    if isinstance(dotted_name, basestring):
+        if ':' not in dotted_name:
+            # backwards compatibility
+            warnings.warn('model should be in form of module.model:User '
+                'and not module.model.User', exceptions.MigrateDeprecationWarning)
+            dotted_name = ':'.join(dotted_name.rsplit('.', 1))
+        return EntryPoint.parse('x=%s' % dotted_name).load(False)
+    else:
+        # Assume it's already loaded.
+        return dotted_name
+
+def asbool(obj):
+    """Do everything to use object as bool"""
+    if isinstance(obj, basestring):
+        obj = obj.strip().lower()
+        if obj in ['true', 'yes', 'on', 'y', 't', '1']:
+            return True
+        elif obj in ['false', 'no', 'off', 'n', 'f', '0']:
+            return False
+        else:
+            raise ValueError("String is not true/false: %r" % obj)
+    if obj in (True, False):
+        return bool(obj)
+    else:
+        raise ValueError("String is not true/false: %r" % obj)
+
+def guess_obj_type(obj):
+    """Do everything to guess object type from string
+    
+    Tries to convert to `int`, `bool` and finally returns if not succeded.
+    
+    .. versionadded: 0.5.4
+    """
+
+    result = None
+
+    try:
+        result = int(obj)
+    except:
+        pass
+
+    if result is None:
+        try:
+            result = asbool(obj)
+        except:
+            pass
+
+    if result is not None:
+        return result
+    else:
+        return obj
+
+@decorator
+def catch_known_errors(f, *a, **kw):
+    """Decorator that catches known api errors
+    
+    .. versionadded: 0.5.4
+    """
+
+    try:
+        return f(*a, **kw)
+    except exceptions.PathFoundError, e:
+        raise exceptions.KnownError("The path %s already exists" % e.args[0])
+
+def construct_engine(engine, **opts):
+    """.. versionadded:: 0.5.4
+
+    Constructs and returns SQLAlchemy engine.
+
+    Currently, there are 2 ways to pass create_engine options to :mod:`migrate.versioning.api` functions:
+
+    :param engine: connection string or a existing engine
+    :param engine_dict: python dictionary of options to pass to `create_engine`
+    :param engine_arg_*: keyword parameters to pass to `create_engine` (evaluated with :func:`migrate.versioning.util.guess_obj_type`)
+    :type engine_dict: dict
+    :type engine: string or Engine instance
+    :type engine_arg_*: string
+    :returns: SQLAlchemy Engine
+
+    .. note::
+
+        keyword parameters override ``engine_dict`` values.
+
+    """
+    if isinstance(engine, Engine):
+        return engine
+    elif not isinstance(engine, basestring):
+        raise ValueError("you need to pass either an existing engine or a database uri")
+
+    # get options for create_engine
+    if opts.get('engine_dict') and isinstance(opts['engine_dict'], dict):
+        kwargs = opts['engine_dict']
+    else:
+        kwargs = dict()
+
+    # DEPRECATED: handle echo the old way
+    echo = asbool(opts.get('echo', False))
+    if echo:
+        warnings.warn('echo=True parameter is deprecated, pass '
+            'engine_arg_echo=True or engine_dict={"echo": True}',
+            exceptions.MigrateDeprecationWarning)
+        kwargs['echo'] = echo
+
+    # parse keyword arguments
+    for key, value in opts.iteritems():
+        if key.startswith('engine_arg_'):
+            kwargs[key[11:]] = guess_obj_type(value)
+
+    log.debug('Constructing engine')
+    # TODO: return create_engine(engine, poolclass=StaticPool, **kwargs)
+    # seems like 0.5.x branch does not work with engine.dispose and staticpool
+    return create_engine(engine, **kwargs)
+
+@decorator
+def with_engine(f, *a, **kw):
+    """Decorator for :mod:`migrate.versioning.api` functions
+    to safely close resources after function usage.
+
+    Passes engine parameters to :func:`construct_engine` and
+    resulting parameter is available as kw['engine'].
+
+    Engine is disposed after wrapped function is executed.
+
+    .. versionadded: 0.6.0
+    """
+    url = a[0]
+    engine = construct_engine(url, **kw)
+
+    try:
+        kw['engine'] = engine
+        return f(*a, **kw)
+    finally:
+        if isinstance(engine, Engine):
+            log.debug('Disposing SQLAlchemy engine %s', engine)
+            engine.dispose()
+
+
+class Memoize:
+    """Memoize(fn) - an instance which acts like fn but memoizes its arguments
+       Will only work on functions with non-mutable arguments
+
+       ActiveState Code 52201
+    """
+    def __init__(self, fn):
+        self.fn = fn
+        self.memo = {}
+
+    def __call__(self, *args):
+        if not self.memo.has_key(args):
+            self.memo[args] = self.fn(*args)
+        return self.memo[args]

rhodecode/lib/dbmigrate/migrate/versioning/util/importpath.py

@@ -0,0 +1,16
+import os
+import sys
+
+def import_path(fullpath):
+    """ Import a file with full path specification. Allows one to
+        import from anywhere, something __import__ does not do. 
+    """
+    # http://zephyrfalcon.org/weblog/arch_d7_2002_08_31.html
+    path, filename = os.path.split(fullpath)
+    filename, ext = os.path.splitext(filename)
+    sys.path.append(path)
+    module = __import__(filename)
+    reload(module) # Might be out of date during tests
+    del sys.path[-1]
+    return module
+

rhodecode/lib/dbmigrate/migrate/versioning/util/keyedinstance.py

@@ -0,0 +1,36
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+class KeyedInstance(object):
+    """A class whose instances have a unique identifier of some sort
+    No two instances with the same unique ID should exist - if we try to create
+    a second instance, the first should be returned. 
+    """
+
+    _instances = dict()
+
+    def __new__(cls, *p, **k):
+        instances = cls._instances
+        clskey = str(cls)
+        if clskey not in instances:
+            instances[clskey] = dict()
+        instances = instances[clskey]
+
+        key = cls._key(*p, **k)
+        if key not in instances:
+            instances[key] = super(KeyedInstance, cls).__new__(cls)
+        return instances[key]
+
+    @classmethod
+    def _key(cls, *p, **k):
+        """Given a unique identifier, return a dictionary key
+        This should be overridden by child classes, to specify which parameters 
+        should determine an object's uniqueness
+        """
+        raise NotImplementedError()
+
+    @classmethod
+    def clear(cls):
+        # Allow cls.clear() as well as uniqueInstance.clear(cls)
+        if str(cls) in cls._instances:
+            del cls._instances[str(cls)]

rhodecode/lib/dbmigrate/migrate/versioning/version.py

@@ -0,0 +1,215
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import re
+import shutil
+import logging
+
+from rhodecode.lib.dbmigrate.migrate import exceptions
+from rhodecode.lib.dbmigrate.migrate.versioning import pathed, script
+
+
+log = logging.getLogger(__name__)
+
+class VerNum(object):
+    """A version number that behaves like a string and int at the same time"""
+
+    _instances = dict()
+
+    def __new__(cls, value):
+        val = str(value)
+        if val not in cls._instances:
+            cls._instances[val] = super(VerNum, cls).__new__(cls)
+        ret = cls._instances[val]
+        return ret
+
+    def __init__(self,value):
+        self.value = str(int(value))
+        if self < 0:
+            raise ValueError("Version number cannot be negative")
+
+    def __add__(self, value):
+        ret = int(self) + int(value)
+        return VerNum(ret)
+
+    def __sub__(self, value):
+        return self + (int(value) * -1)
+
+    def __cmp__(self, value):
+        return int(self) - int(value)
+
+    def __repr__(self):
+        return "<VerNum(%s)>" % self.value
+
+    def __str__(self):
+        return str(self.value)
+
+    def __int__(self):
+        return int(self.value)
+
+
+class Collection(pathed.Pathed):
+    """A collection of versioning scripts in a repository"""
+
+    FILENAME_WITH_VERSION = re.compile(r'^(\d{3,}).*')
+
+    def __init__(self, path):
+        """Collect current version scripts in repository
+        and store them in self.versions
+        """
+        super(Collection, self).__init__(path)
+        
+        # Create temporary list of files, allowing skipped version numbers.
+        files = os.listdir(path)
+        if '1' in files:
+            # deprecation
+            raise Exception('It looks like you have a repository in the old '
+                'format (with directories for each version). '
+                'Please convert repository before proceeding.')
+
+        tempVersions = dict()
+        for filename in files:
+            match = self.FILENAME_WITH_VERSION.match(filename)
+            if match:
+                num = int(match.group(1))
+                tempVersions.setdefault(num, []).append(filename)
+            else:
+                pass  # Must be a helper file or something, let's ignore it.
+
+        # Create the versions member where the keys
+        # are VerNum's and the values are Version's.
+        self.versions = dict()
+        for num, files in tempVersions.items():
+            self.versions[VerNum(num)] = Version(num, path, files)
+
+    @property
+    def latest(self):
+        """:returns: Latest version in Collection"""
+        return max([VerNum(0)] + self.versions.keys())
+
+    def create_new_python_version(self, description, **k):
+        """Create Python files for new version"""
+        ver = self.latest + 1
+        extra = str_to_filename(description)
+
+        if extra:
+            if extra == '_':
+                extra = ''
+            elif not extra.startswith('_'):
+                extra = '_%s' % extra
+
+        filename = '%03d%s.py' % (ver, extra)
+        filepath = self._version_path(filename)
+
+        script.PythonScript.create(filepath, **k)
+        self.versions[ver] = Version(ver, self.path, [filename])
+        
+    def create_new_sql_version(self, database, **k):
+        """Create SQL files for new version"""
+        ver = self.latest + 1
+        self.versions[ver] = Version(ver, self.path, [])
+
+        # Create new files.
+        for op in ('upgrade', 'downgrade'):
+            filename = '%03d_%s_%s.sql' % (ver, database, op)
+            filepath = self._version_path(filename)
+            script.SqlScript.create(filepath, **k)
+            self.versions[ver].add_script(filepath)
+        
+    def version(self, vernum=None):
+        """Returns latest Version if vernum is not given.
+        Otherwise, returns wanted version"""
+        if vernum is None:
+            vernum = self.latest
+        return self.versions[VerNum(vernum)]
+
+    @classmethod
+    def clear(cls):
+        super(Collection, cls).clear()
+
+    def _version_path(self, ver):
+        """Returns path of file in versions repository"""
+        return os.path.join(self.path, str(ver))
+
+
+class Version(object):
+    """A single version in a collection
+    :param vernum: Version Number 
+    :param path: Path to script files
+    :param filelist: List of scripts
+    :type vernum: int, VerNum
+    :type path: string
+    :type filelist: list
+    """
+
+    def __init__(self, vernum, path, filelist):
+        self.version = VerNum(vernum)
+
+        # Collect scripts in this folder
+        self.sql = dict()
+        self.python = None
+
+        for script in filelist:
+            self.add_script(os.path.join(path, script))
+    
+    def script(self, database=None, operation=None):
+        """Returns SQL or Python Script"""
+        for db in (database, 'default'):
+            # Try to return a .sql script first
+            try:
+                return self.sql[db][operation]
+            except KeyError:
+                continue  # No .sql script exists
+
+        # TODO: maybe add force Python parameter?
+        ret = self.python
+
+        assert ret is not None, \
+            "There is no script for %d version" % self.version
+        return ret
+
+    def add_script(self, path):
+        """Add script to Collection/Version"""
+        if path.endswith(Extensions.py):
+            self._add_script_py(path)
+        elif path.endswith(Extensions.sql):
+            self._add_script_sql(path)
+
+    SQL_FILENAME = re.compile(r'^(\d+)_([^_]+)_([^_]+).sql')
+
+    def _add_script_sql(self, path):
+        basename = os.path.basename(path)
+        match = self.SQL_FILENAME.match(basename)
+
+        if match:
+            version, dbms, op = match.group(1), match.group(2), match.group(3)
+        else:
+            raise exceptions.ScriptError(
+                "Invalid SQL script name %s " % basename + \
+                "(needs to be ###_database_operation.sql)")
+
+        # File the script into a dictionary
+        self.sql.setdefault(dbms, {})[op] = script.SqlScript(path)
+
+    def _add_script_py(self, path):
+        if self.python is not None:
+            raise exceptions.ScriptError('You can only have one Python script '
+                'per version, but you have: %s and %s' % (self.python, path))
+        self.python = script.PythonScript(path)
+
+
+class Extensions:
+    """A namespace for file extensions"""
+    py = 'py'
+    sql = 'sql'
+
+def str_to_filename(s):
+    """Replaces spaces, (double and single) quotes
+    and double underscores to underscores
+    """
+
+    s = s.replace(' ', '_').replace('"', '_').replace("'", '_').replace(".", "_")
+    while '__' in s:
+        s = s.replace('__', '_')
+    return s

rhodecode/lib/dbmigrate/versions/001_initial_release.py

@@ -0,0 +1,237
+#==============================================================================
+# DB INITIAL MODEL
+#==============================================================================
+import logging
+import datetime
+
+from sqlalchemy import *
+from sqlalchemy.exc import DatabaseError
+from sqlalchemy.orm import relation, backref, class_mapper
+from sqlalchemy.orm.session import Session
+from rhodecode.model.meta import Base
+
+from rhodecode.lib.dbmigrate.migrate import *
+
+log = logging.getLogger(__name__)
+
+class BaseModel(object):
+
+    @classmethod
+    def _get_keys(cls):
+        """return column names for this model """
+        return class_mapper(cls).c.keys()
+
+    def get_dict(self):
+        """return dict with keys and values corresponding 
+        to this model data """
+
+        d = {}
+        for k in self._get_keys():
+            d[k] = getattr(self, k)
+        return d
+
+    def get_appstruct(self):
+        """return list with keys and values tupples corresponding 
+        to this model data """
+
+        l = []
+        for k in self._get_keys():
+            l.append((k, getattr(self, k),))
+        return l
+
+    def populate_obj(self, populate_dict):
+        """populate model with data from given populate_dict"""
+
+        for k in self._get_keys():
+            if k in populate_dict:
+                setattr(self, k, populate_dict[k])
+
+class RhodeCodeSettings(Base, BaseModel):
+    __tablename__ = 'rhodecode_settings'
+    __table_args__ = (UniqueConstraint('app_settings_name'), {'useexisting':True})
+    app_settings_id = Column("app_settings_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    app_settings_name = Column("app_settings_name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    app_settings_value = Column("app_settings_value", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+
+    def __init__(self, k, v):
+        self.app_settings_name = k
+        self.app_settings_value = v
+
+    def __repr__(self):
+        return "<RhodeCodeSetting('%s:%s')>" % (self.app_settings_name,
+                                                self.app_settings_value)
+
+class RhodeCodeUi(Base, BaseModel):
+    __tablename__ = 'rhodecode_ui'
+    __table_args__ = {'useexisting':True}
+    ui_id = Column("ui_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    ui_section = Column("ui_section", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    ui_key = Column("ui_key", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    ui_value = Column("ui_value", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    ui_active = Column("ui_active", Boolean(), nullable=True, unique=None, default=True)
+
+
+class User(Base, BaseModel):
+    __tablename__ = 'users'
+    __table_args__ = (UniqueConstraint('username'), UniqueConstraint('email'), {'useexisting':True})
+    user_id = Column("user_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    username = Column("username", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    password = Column("password", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    active = Column("active", Boolean(), nullable=True, unique=None, default=None)
+    admin = Column("admin", Boolean(), nullable=True, unique=None, default=False)
+    name = Column("name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    lastname = Column("lastname", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    email = Column("email", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    last_login = Column("last_login", DateTime(timezone=False), nullable=True, unique=None, default=None)
+    is_ldap = Column("is_ldap", Boolean(), nullable=False, unique=None, default=False)
+
+    user_log = relation('UserLog', cascade='all')
+    user_perms = relation('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id", cascade='all')
+
+    repositories = relation('Repository')
+    user_followers = relation('UserFollowing', primaryjoin='UserFollowing.follows_user_id==User.user_id', cascade='all')
+
+    @property
+    def full_contact(self):
+        return '%s %s <%s>' % (self.name, self.lastname, self.email)
+
+    def __repr__(self):
+        return "<User('id:%s:%s')>" % (self.user_id, self.username)
+
+    def update_lastlogin(self):
+        """Update user lastlogin"""
+
+        try:
+            session = Session.object_session(self)
+            self.last_login = datetime.datetime.now()
+            session.add(self)
+            session.commit()
+            log.debug('updated user %s lastlogin', self.username)
+        except (DatabaseError,):
+            session.rollback()
+
+
+class UserLog(Base, BaseModel):
+    __tablename__ = 'user_logs'
+    __table_args__ = {'useexisting':True}
+    user_log_id = Column("user_log_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
+    repository_id = Column("repository_id", Integer(length=None, convert_unicode=False, assert_unicode=None), ForeignKey(u'repositories.repo_id'), nullable=False, unique=None, default=None)
+    repository_name = Column("repository_name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    user_ip = Column("user_ip", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    action = Column("action", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    action_date = Column("action_date", DateTime(timezone=False), nullable=True, unique=None, default=None)
+
+    user = relation('User')
+    repository = relation('Repository')
+
+class Repository(Base, BaseModel):
+    __tablename__ = 'repositories'
+    __table_args__ = (UniqueConstraint('repo_name'), {'useexisting':True},)
+    repo_id = Column("repo_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    repo_name = Column("repo_name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
+    repo_type = Column("repo_type", String(length=None, convert_unicode=False, assert_unicode=None), nullable=False, unique=False, default=None)
+    user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=False, default=None)
+    private = Column("private", Boolean(), nullable=True, unique=None, default=None)
+    enable_statistics = Column("statistics", Boolean(), nullable=True, unique=None, default=True)
+    description = Column("description", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    fork_id = Column("fork_id", Integer(), ForeignKey(u'repositories.repo_id'), nullable=True, unique=False, default=None)
+
+    user = relation('User')
+    fork = relation('Repository', remote_side=repo_id)
+    repo_to_perm = relation('RepoToPerm', cascade='all')
+    stats = relation('Statistics', cascade='all', uselist=False)
+
+    repo_followers = relation('UserFollowing', primaryjoin='UserFollowing.follows_repo_id==Repository.repo_id', cascade='all')
+
+
+    def __repr__(self):
+        return "<Repository('%s:%s')>" % (self.repo_id, self.repo_name)
+
+class Permission(Base, BaseModel):
+    __tablename__ = 'permissions'
+    __table_args__ = {'useexisting':True}
+    permission_id = Column("permission_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    permission_name = Column("permission_name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    permission_longname = Column("permission_longname", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+
+    def __repr__(self):
+        return "<Permission('%s:%s')>" % (self.permission_id, self.permission_name)
+
+class RepoToPerm(Base, BaseModel):
+    __tablename__ = 'repo_to_perm'
+    __table_args__ = (UniqueConstraint('user_id', 'repository_id'), {'useexisting':True})
+    repo_to_perm_id = Column("repo_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
+    permission_id = Column("permission_id", Integer(), ForeignKey(u'permissions.permission_id'), nullable=False, unique=None, default=None)
+    repository_id = Column("repository_id", Integer(), ForeignKey(u'repositories.repo_id'), nullable=False, unique=None, default=None)
+
+    user = relation('User')
+    permission = relation('Permission')
+    repository = relation('Repository')
+
+class UserToPerm(Base, BaseModel):
+    __tablename__ = 'user_to_perm'
+    __table_args__ = (UniqueConstraint('user_id', 'permission_id'), {'useexisting':True})
+    user_to_perm_id = Column("user_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
+    permission_id = Column("permission_id", Integer(), ForeignKey(u'permissions.permission_id'), nullable=False, unique=None, default=None)
+
+    user = relation('User')
+    permission = relation('Permission')
+
+class Statistics(Base, BaseModel):
+    __tablename__ = 'statistics'
+    __table_args__ = (UniqueConstraint('repository_id'), {'useexisting':True})
+    stat_id = Column("stat_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    repository_id = Column("repository_id", Integer(), ForeignKey(u'repositories.repo_id'), nullable=False, unique=True, default=None)
+    stat_on_revision = Column("stat_on_revision", Integer(), nullable=False)
+    commit_activity = Column("commit_activity", LargeBinary(), nullable=False)#JSON data
+    commit_activity_combined = Column("commit_activity_combined", LargeBinary(), nullable=False)#JSON data
+    languages = Column("languages", LargeBinary(), nullable=False)#JSON data
+
+    repository = relation('Repository', single_parent=True)
+
+class UserFollowing(Base, BaseModel):
+    __tablename__ = 'user_followings'
+    __table_args__ = (UniqueConstraint('user_id', 'follows_repository_id'),
+                      UniqueConstraint('user_id', 'follows_user_id')
+                      , {'useexisting':True})
+
+    user_following_id = Column("user_following_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
+    follows_repo_id = Column("follows_repository_id", Integer(), ForeignKey(u'repositories.repo_id'), nullable=True, unique=None, default=None)
+    follows_user_id = Column("follows_user_id", Integer(), ForeignKey(u'users.user_id'), nullable=True, unique=None, default=None)
+
+    user = relation('User', primaryjoin='User.user_id==UserFollowing.user_id')
+
+    follows_user = relation('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')
+    follows_repository = relation('Repository')
+
+
+class CacheInvalidation(Base, BaseModel):
+    __tablename__ = 'cache_invalidation'
+    __table_args__ = (UniqueConstraint('cache_key'), {'useexisting':True})
+    cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    cache_key = Column("cache_key", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    cache_args = Column("cache_args", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False)
+
+
+    def __init__(self, cache_key, cache_args=''):
+        self.cache_key = cache_key
+        self.cache_args = cache_args
+        self.cache_active = False
+
+    def __repr__(self):
+        return "<CacheInvalidation('%s:%s')>" % (self.cache_id, self.cache_key)
+
+
+def upgrade(migrate_engine):
+    # Upgrade operations go here. Don't create your own engine; bind migrate_engine
+    # to your metadata
+    Base.metadata.create_all(bind=migrate_engine, checkfirst=False)
+
+def downgrade(migrate_engine):
+    # Operations to reverse the above upgrade go here.
+    Base.metadata.drop_all(bind=migrate_engine, checkfirst=False)

rhodecode/lib/dbmigrate/versions/002_version_1_1_0.py

@@ -0,0 +1,127
+import logging
+import datetime
+
+from sqlalchemy import *
+from sqlalchemy.exc import DatabaseError
+from sqlalchemy.orm import relation, backref, class_mapper
+from sqlalchemy.orm.session import Session
+from rhodecode.model.meta import Base
+from rhodecode.model.db import BaseModel
+
+from rhodecode.lib.dbmigrate.migrate import *
+
+log = logging.getLogger(__name__)
+
+def upgrade(migrate_engine):
+    """ Upgrade operations go here. 
+    Don't create your own engine; bind migrate_engine to your metadata
+    """
+
+    #==========================================================================
+    # Upgrade of `users` table
+    #==========================================================================
+    tblname = 'users'
+    tbl = Table(tblname, MetaData(bind=migrate_engine), autoload=True,
+                    autoload_with=migrate_engine)
+
+    #ADD is_ldap column
+    is_ldap = Column("is_ldap", Boolean(), nullable=True,
+                     unique=None, default=False)
+    is_ldap.create(tbl, populate_default=True)
+    is_ldap.alter(nullable=False)
+
+    #==========================================================================
+    # Upgrade of `user_logs` table
+    #==========================================================================    
+
+    tblname = 'users'
+    tbl = Table(tblname, MetaData(bind=migrate_engine), autoload=True,
+                    autoload_with=migrate_engine)
+
+    #ADD revision column
+    revision = Column('revision', TEXT(length=None, convert_unicode=False,
+                                       assert_unicode=None),
+                      nullable=True, unique=None, default=None)
+    revision.create(tbl)
+
+
+
+    #==========================================================================
+    # Upgrade of `repositories` table
+    #==========================================================================    
+    tblname = 'repositories'
+    tbl = Table(tblname, MetaData(bind=migrate_engine), autoload=True,
+                    autoload_with=migrate_engine)
+
+    #ADD repo_type column#
+    repo_type = Column("repo_type", String(length=None, convert_unicode=False,
+                                           assert_unicode=None),
+                       nullable=True, unique=False, default='hg')
+
+    repo_type.create(tbl, populate_default=True)
+    #repo_type.alter(nullable=False)
+
+    #ADD statistics column#
+    enable_statistics = Column("statistics", Boolean(), nullable=True,
+                               unique=None, default=True)
+    enable_statistics.create(tbl)
+
+    #==========================================================================
+    # Add table `user_followings`
+    #==========================================================================
+    tblname = 'user_followings'
+
+    class UserFollowing(Base, BaseModel):
+        __tablename__ = 'user_followings'
+        __table_args__ = (UniqueConstraint('user_id', 'follows_repository_id'),
+                          UniqueConstraint('user_id', 'follows_user_id')
+                          , {'useexisting':True})
+
+        user_following_id = Column("user_following_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+        user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
+        follows_repo_id = Column("follows_repository_id", Integer(), ForeignKey(u'repositories.repo_id'), nullable=True, unique=None, default=None)
+        follows_user_id = Column("follows_user_id", Integer(), ForeignKey(u'users.user_id'), nullable=True, unique=None, default=None)
+
+        user = relation('User', primaryjoin='User.user_id==UserFollowing.user_id')
+
+        follows_user = relation('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')
+        follows_repository = relation('Repository')
+
+    Base.metadata.tables[tblname].create(migrate_engine)
+
+    #==========================================================================
+    # Add table `cache_invalidation`
+    #==========================================================================
+    tblname = 'cache_invalidation'
+
+    class CacheInvalidation(Base, BaseModel):
+        __tablename__ = 'cache_invalidation'
+        __table_args__ = (UniqueConstraint('cache_key'), {'useexisting':True})
+        cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+        cache_key = Column("cache_key", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+        cache_args = Column("cache_args", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+        cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False)
+
+
+        def __init__(self, cache_key, cache_args=''):
+            self.cache_key = cache_key
+            self.cache_args = cache_args
+            self.cache_active = False
+
+        def __repr__(self):
+            return "<CacheInvalidation('%s:%s')>" % (self.cache_id, self.cache_key)
+
+    Base.metadata.tables[tblname].create(migrate_engine)
+
+    return
+
+
+
+
+
+
+def downgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+
+

rhodecode/lib/dbmigrate/versions/__init__.py

@@ -0,0 +1,26
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.lib.dbmigrate.versions.__init__
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Package containing new versions of database models
+    
+    :created_on: Dec 11, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your opinion) any later version of the license.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.

rhodecode/lib/exceptions.py

@@ -0,0 +1,32
+#!/usr/bin/env python
+# encoding: utf-8
+# Custom Exceptions modules
+# Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your opinion) any later version of the license.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+"""
+Created on Nov 17, 2010
+Custom Exceptions modules
+@author: marcink
+"""
+
+class LdapUsernameError(Exception):pass
+class LdapPasswordError(Exception):pass
+class LdapConnectionError(Exception):pass
+class LdapImportError(Exception):pass
+
+class DefaultUserException(Exception):pass
+class UserOwnsReposException(Exception):pass

.hgignore

@@ -1,5 +1,11
+syntax: glob
+*.pyc
+*.swp
 
 syntax: regexp
+^build
+^docs/build/
+^docs/_build/
 ^data$
 ^\.settings$
 ^\.project$
@@ -7,4 +13,4 syntax: regexp
 ^rhodecode\.db$
 ^test\.db$
 ^repositories\.config$
-^RhodeCode\.egg-info$
  No newline at end of file
+^RhodeCode\.egg-info$

MANIFEST.in

@@ -1,4 +1,5
 include rhodecode/config/deployment.ini_tmpl
+include rhodecode/lib/dbmigrate/migrate.cfg
 
 include README.rst
 recursive-include rhodecode/i18n/ *
@@ -7,7 +8,7 recursive-include rhodecode/i18n/ *
 recursive-include rhodecode/public/css *
 recursive-include rhodecode/public/images *
 #js
-include rhodecode/public/js/yui2.js
+include rhodecode/public/js/yui2a.js
 include rhodecode/public/js/excanvas.min.js
 include rhodecode/public/js/yui.flot.js
 include rhodecode/public/js/graph.js

README.rst

@@ -1,23 +1,25
 
-RhodeCode (RhodiumCode)
-=======================
+=================================================
+Welcome to RhodeCode (RhodiumCode) documentation!
+=================================================
 
-``RhodeCode`` (formerly hg-app) is Pylons based repository management and 
-serving for mercurial_. It's similar to github or bitbucket, but it's suppose to run
-as standalone app, it's open source and focuses more on restricted access to repositories
-There's no default free access to RhodeCode You have to create an account in order
-to use the application. It's powered by vcs_ library that we created to handle
-many various version control systems.
+``RhodeCode`` (formerly hg-app) is Pylons framework based Mercurial repository 
+browser/management with build in push/pull server and full text search.
+It works on http/https, has build in permission/authentication(+ldap) features 
+It's similar to github or bitbucket, but it's suppose to run as standalone 
+hosted application, it's open source and focuses more on restricted access to 
+repositories. It's powered by vcs_ library that me and Lukasz Balcerzak created 
+to handle many various version control systems.
 
 RhodeCode uses `Semantic Versioning <http://semver.org/>`_
 
-
 RhodeCode demo
 --------------
 
 http://hg.python-works.com
 
-The default access is
+The default access is anonymous but You can login to administrative account
+using those credentials
 
 - username: demo
 - password: demo
@@ -25,14 +27,14 The default access is
 Source code
 -----------
 
-Source code is along with issue tracker is available at
+The most up to date sources can be obtained from my own RhodeCode instance
+https://rhodecode.org 
+
+Rarely updated source code and issue tracker is available at bitbcuket
 http://bitbucket.org/marcinkuzminski/rhodecode
 
-Also a source codes can be obtained from demo rhodecode instance
-http://hg.python-works.com/rhodecode/summary
-
-Instalation
------------
+Installation
+------------
 
  Please visit http://packages.python.org/RhodeCode/installation.html
 
@@ -40,41 +42,49 Instalation
 Features
 --------
 
-- Has it's own middleware to handle mercurial_ protocol request. Each request 
-  can be logged and authenticated. Runs on threads unlikely to hgweb You can 
-  make multiple pulls/pushes simultaneous. Supports http/https
-- Full permissions and authentication per project private/read/write/admin. 
-  One account for web interface and mercurial_ push/pull/clone.
+- Has it's own middleware to handle mercurial_ protocol request. 
+  Each request can be logged and authenticated. Runs on threads unlikely to 
+  hgweb. You can make multiple pulls/pushes simultaneous. Supports http/https 
+  and ldap
+- Full permissions (private/read/write/admin) and authentication per project. 
+  One account for web interface and mercurial_ push/pull/clone operations.
 - Mako templates let's you customize look and feel of application.
 - Beautiful diffs, annotations and source codes all colored by pygments.
 - Mercurial_ branch graph and yui-flot powered graphs with zooming and statistics
-- Admin interface with user/permission management. User activity journal logs
-  pulls, pushes, forks,registrations. Possible to disable built in hooks
+- Admin interface with user/permission management. Admin activity journal, logs
+  pulls, pushes, forks, registrations and other actions made by all users.
 - Server side forks, it's possible to fork a project and hack it free without
-  breaking the main.   
-- Full text search on source codes, search on file names. All powered by whoosh
-  and build in indexing daemons
+  breaking the main repository.
+- Full text search powered by Whoosh on source codes, and file names.
+  Build in indexing daemons, with optional incremental index build
   (no external search servers required all in one application)
-- Rss / atom feeds, gravatar support, download sources as zip/tarballs  
+- Setup project descriptions and info inside built in db for easy, non 
+  file-system operations
+- Inteligent cache with invalidation after push or project change, provides high 
+  performance and always up to date data.    
+- Rss / atom feeds, gravatar support, download sources as zip/tar/gz
 - Async tasks for speed and performance using celery_ (works without them too)  
 - Backup scripts can do backup of whole app and send it over scp to desired 
   location
-- Setup project descriptions and info inside built in db for easy, non 
-  file-system operations
-- Added cache with invalidation on push/repo management for high performance and
-  always up to date data. 
-- Based on pylons 1.0 / sqlalchemy 0.6 / sqlite
+- Based on pylons / sqlalchemy / sqlite / whoosh / vcs
+
+
+.. include:: ./docs/screenshots.rst
 
 
-Incoming
---------
+Incoming / Plans
+----------------
 
+- project grouping
+- User groups/teams
 - code review (probably based on hg-review)
-- full git_ support, with push/pull server
+- full git_ support, with push/pull server (currently in beta tests)
+- redmine integration
+- public accessible activity feeds
 - commit based build in wiki system
 - clone points and cloning from remote repositories into rhodecode 
   (git_ and mercurial_)
-- some cache optimizations
+- more statistics and graph (global annotation + some more statistics)
 - other cools stuff that i can figure out (or You can help me figure out)
 
 License
@@ -83,8 +93,18 License
 ``rhodecode`` is released under GPL_ license.
 
 
-Documentation
--------------
+Mailing group Q&A
+-----------------
+
+join the `Google group <http://groups.google.com/group/rhodecode>`_
+
+open an issue at `issue tracker <http://bitbucket.org/marcinkuzminski/rhodecode/issues>`_
+
+join #rhodecode on FreeNode (irc.freenode.net)
+or use http://webchat.freenode.net/?channels=rhodecode for web access to irc.
+
+Online documentation
+--------------------
 
  Online documentation for current version is available at
  http://packages.python.org/RhodeCode/.
@@ -92,13 +112,3 Documentation
 
    make html
 
-.. _virtualenv: http://pypi.python.org/pypi/virtualenv
-.. _python: http://www.python.org/
-.. _django: http://www.djangoproject.com/
-.. _mercurial: http://mercurial.selenic.com/
-.. _subversion: http://subversion.tigris.org/
-.. _git: http://git-scm.com/
-.. _celery: http://celeryproject.org/
-.. _Sphinx: http://sphinx.pocoo.org/
-.. _GPL: http://www.gnu.org/licenses/gpl.html
-.. _vcs: http://pypi.python.org/pypi/vcs
  No newline at end of file

development.ini

@@ -1,6 +1,6
 ################################################################################
 ################################################################################
-# rhodecode - Pylons environment configuration                                 #
+# RhodeCode - Pylons environment configuration                                 #
 #                                                                              # 
 # The %(here)s variable will be replaced with the parent directory of this file#
 ################################################################################
@@ -10,7 +10,7 debug = true
 ################################################################################
 ## Uncomment and replace with the address which should receive                ## 
 ## any error reports after application crash								  ##
-## Additionally those settings will be used by rhodecode mailing system       ##
+## Additionally those settings will be used by RhodeCode mailing system       ##
 ################################################################################
 #email_to = admin@localhost
 #error_email_from = paste_error@localhost
@@ -21,20 +21,21 debug = true
 #smtp_username = 
 #smtp_password =
 #smtp_port = 
-#smtp_use_tls = 
+#smtp_use_tls = false
+#smtp_use_ssl = true
 
 [server:main]
 ##nr of threads to spawn
 threadpool_workers = 5
 
-##max request before
+##max request before thread respawn
 threadpool_max_requests = 6
 
 ##option to use threads of process
 use_threadpool = false
 
 use = egg:Paste#http
-host = 127.0.0.1
+host = 0.0.0.0
 port = 5000
 
 [app:main]
@@ -43,6 +44,35 full_stack = true
 static_files = true
 lang=en
 cache_dir = %(here)s/data
+index_dir = %(here)s/data/index
+cut_off_limit = 256000
+
+####################################
+###        CELERY CONFIG        ####
+####################################
+use_celery = false
+broker.host = localhost
+broker.vhost = rabbitmqhost
+broker.port = 5672
+broker.user = rabbitmq
+broker.password = qweqwe
+
+celery.imports = rhodecode.lib.celerylib.tasks
+
+celery.result.backend = amqp
+celery.result.dburi = amqp://
+celery.result.serialier = json
+
+#celery.send.task.error.emails = true
+#celery.amqp.task.result.expires = 18000
+
+celeryd.concurrency = 2
+#celeryd.log.file = celeryd.log
+celeryd.log.level = debug
+celeryd.max.tasks.per.child = 3
+
+#tasks will never be sent to the queue, but executed locally instead.
+celery.always.eager = false
 
 ####################################
 ###         BEAKER CACHE        ####
@@ -60,9 +90,8 beaker.cache.short_term.expire=60
 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
 
-
 beaker.cache.sql_cache_short.type=memory
-beaker.cache.sql_cache_short.expire=5
+beaker.cache.sql_cache_short.expire=10
 
 beaker.cache.sql_cache_med.type=memory
 beaker.cache.sql_cache_med.expire=360
@@ -74,7 +103,7 beaker.cache.sql_cache_long.expire=3600
 ###       BEAKER SESSION        ####
 ####################################
 ## Type of storage used for the session, current types are 
-## "dbm", "file", "memcached", "database", and "memory". 
+## dbm, file, memcached, database, and memory. 
 ## The storage uses the Container API 
 ##that is also used by the cache system.
 beaker.session.type = file
@@ -116,7 +145,7 sqlalchemy.convert_unicode = true
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
-keys = root, routes, rhodecode, sqlalchemy
+keys = root, routes, rhodecode, sqlalchemy,beaker,templates
 
 [handlers]
 keys = console
@@ -136,6 +165,19 level = DEBUG
 handlers = console
 qualname = routes.middleware
 # "level = DEBUG" logs the route matched and routing variables.
+propagate = 0
+
+[logger_beaker]
+level = ERROR
+handlers = console
+qualname = beaker.container
+propagate = 0
+
+[logger_templates]
+level = INFO
+handlers = console
+qualname = pylons.templating
+propagate = 0
 
 [logger_rhodecode]
 level = DEBUG

docs/changelog.rst

@@ -3,22 +3,88
 Changelog
 =========
 
-1.0.2 (**2010-11-XX**)
+1.1.0 (**2010-12-18**)
 ----------------------
 
+:status: in-progress
+:branch: beta
+
+news
+++++
+
+- rewrite of internals for vcs >=0.1.10
+- uses mercurial 1.7 with dotencode disabled for maintaining compatibility 
+  with older clients
+- anonymous access, authentication via ldap
+- performance upgrade for cached repos list - each repository has it's own 
+  cache that's invalidated when needed.
+- performance upgrades on repositories with large amount of commits (20K+)
+- main page quick filter for filtering repositories
+- user dashboards with ability to follow chosen repositories actions
+- sends email to admin on new user registration
+- added cache/statistics reset options into repository settings
+- more detailed action logger (based on hooks) with pushed changesets lists
+  and options to disable those hooks from admin panel
+- introduced new enhanced changelog for merges that shows more accurate results
+- new improved and faster code stats (based on pygments lexers mapping tables, 
+  showing up to 10 trending sources for each repository. Additionally stats
+  can be disabled in repository settings.
+- gui optimizations, fixed application width to 1024px
+- added cut off (for large files/changesets) limit into config files
+- whoosh, celeryd, upgrade moved to paster command
+- other than sqlite database backends can be used
+
+fixes
++++++
+
+- fixes #61 forked repo was showing only after cache expired
+- fixes #76 no confirmation on user deletes
+- fixes #66 Name field misspelled
+- fixes #72 block user removal when he owns repositories
+- fixes #69 added password confirmation fields
+- fixes #87 RhodeCode crashes occasionally on updating repository owner
+- fixes #82 broken annotations on files with more than 1 blank line at the end
+- a lot of fixes and tweaks for file browser
+- fixed detached session issues
+- fixed when user had no repos he would see all repos listed in my account
+- fixed ui() instance bug when global hgrc settings was loaded for server 
+  instance and all hgrc options were merged with our db ui() object
+- numerous small bugfixes
+ 
+(special thanks for TkSoh for detailed feedback)
+
+
+1.0.2 (**2010-11-12**)
+----------------------
+
+news
+++++
+
+- tested under python2.7
+- bumped sqlalchemy and celery versions
+
+fixes
++++++
+
 - fixed #59 missing graph.js
 - fixed repo_size crash when repository had broken symlinks
 - fixed python2.5 crashes.
-- tested under python2.7
-- bumped sqlalcehmy and celery versions
+
 
 1.0.1 (**2010-11-10**)
 ----------------------
 
+news
+++++
+
+- small css updated
+
+fixes
++++++
+
 - fixed #53 python2.5 incompatible enumerate calls
 - fixed #52 disable mercurial extension for web
 - fixed #51 deleting repositories don't delete it's dependent objects
-- small css updated
 
 
 1.0.0 (**2010-11-02**)
@@ -52,3 +118,4 1.0.0rc2 (**2010-10-11**)
 - Disabled dirsize in file browser, it's causing nasty bug when dir renames 
   occure. After vcs is fixed it'll be put back again.
 - templating/css rewrites, optimized css.
+

docs/conf.py

@@ -16,7 +16,7 import sys, os
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.insert(0, os.path.abspath('.'))
+sys.path.insert(0, os.path.abspath('..'))
 
 # -- General configuration -----------------------------------------------------
 

docs/index.rst

@@ -1,107 +1,43
 .. _index:
 
-Welcome to RhodeCode (RhodiumCode) documentation!
-=================================================
-
-``RhodeCode`` (formerly hg-app) is Pylons based repository management and 
-serving for mercurial_. It's similar to github or bitbucket, but it's suppose to run
-as standalone app, it's open source and focuses more on restricted access to repositories
-There's no default free access to RhodeCode You have to create an account in order
-to use the application. It's powered by vcs_ library that we created to handle
-many various version control systems.
-
-RhodeCode uses `Semantic Versioning <http://semver.org/>`_
-
-
-RhodeCode demo
---------------
-
-http://hg.python-works.com
-
-The default access is
-
-- username: demo
-- password: demo
-
-Source code
------------
-
-Source code is along with issue tracker is available at
-http://bitbucket.org/marcinkuzminski/rhodecode
-
-Also a source codes can be obtained from demo rhodecode instance
-http://hg.python-works.com/rhodecode/summary
-
-Features
---------
-
-- Has it's own middleware to handle mercurial_ protocol request. Each request 
-  can be logged and authenticated. Runs on threads unlikely to hgweb You can 
-  make multiple pulls/pushes simultaneous. Supports http/https
-- Full permissions and authentication per project private/read/write/admin. 
-  One account for web interface and mercurial_ push/pull/clone.
-- Mako templates let's you customize look and feel of application.
-- Beautiful diffs, annotations and source codes all colored by pygments.
-- Mercurial_ branch graph and yui-flot powered graphs with zooming and statistics
-- Admin interface with user/permission management. User activity journal logs
-  pulls, pushes, forks,registrations. Possible to disable built in hooks
-- Server side forks, it's possible to fork a project and hack it free without
-  breaking the main.   
-- Full text search on source codes, search on file names. All powered by whoosh
-  and build in indexing daemons
-  (no external search servers required all in one application)
-- Rss / atom feeds, gravatar support, download sources as zip/tarballs  
-- Async tasks for speed and performance using celery_ (works without them too)  
-- Backup scripts can do backup of whole app and send it over scp to desired 
-  location
-- Setup project descriptions and info inside built in db for easy, non 
-  file-system operations
-- Added cache with invalidation on push/repo management for high performance and
-  always up to date data. 
-- Based on pylons 1.0 / sqlalchemy 0.6 / sqlite
-
-
-.. figure::  images/screenshot1_main_page.png
-   :align:   left
-
-   Main page of RhodeCode
-
-.. figure::  images/screenshot2_summary_page.png
-   :align:   left
-
-   Summary page
-
-
-Incoming
---------
-
-- code review (probably based on hg-review)
-- full git_ support, with push/pull server
-- commit based build in wiki system
-- clone points and cloning from remote repositories into rhodecode 
-  (git_ and mercurial_)
-- more statistics and graph (global annotation + some more statistics)
-- user customized activity dashboards
-- some cache optimizations
-- other cools stuff that i can figure out (or You can help me figure out)
-
-License
--------
-
-``rhodecode`` is released under GPL_ license.
-
+.. include:: ./../README.rst
 
 Documentation
 -------------
 
+**Installation:**
+
 .. toctree::
    :maxdepth: 1
 
    installation
+   setup
    upgrade
-   setup
+   
+**Usage**
+
+.. toctree::
+   :maxdepth: 1
+
+   enable_git
+   statistics
+   
+**Develop**
+
+.. toctree::
+   :maxdepth: 1
+   
+   contributing
    changelog
 
+**API**
+
+.. toctree::
+   :maxdepth: 2
+
+   api/index
+   
+
 Other topics
 ------------
 

docs/installation.rst

@@ -5,31 +5,20 Installation
 
 ``RhodeCode`` is written entirely in Python, but in order to use it's full
 potential there are some third-party requirements. When RhodeCode is used 
-together with celery_ You have to install some kind of message broker,
+together with celery You have to install some kind of message broker,
 recommended one is rabbitmq_ to make the async tasks work.
 
 Of course RhodeCode works in sync mode also, then You don't have to install
 any third party apps. Celery_ will give You large speed improvement when using
-many big repositories. If You plan to use it for 5 or 10 small repositories, it
+many big repositories. If You plan to use it for 7 or 10 small repositories, it
 will work just fine without celery running.
    
-After You decide to Run it with celery make sure You run celeryd and
-message broker together with the application.   
-
-Requirements for Celery
------------------------
-
-**Message Broker** 
-
-- preferred is `RabbitMq <http://www.rabbitmq.com/>`_
-- possible other is `Redis <http://code.google.com/p/redis/>`_
-
-For installation instructions You can visit: 
-http://ask.github.com/celery/getting-started/index.html
-It's very nice tutorial how to start celery_ with rabbitmq_
+After You decide to Run it with celery make sure You run celeryd using paster
+and message broker together with the application.   
 
 Install from Cheese Shop
 ------------------------
+Rhodecode requires python 2.x greater than version 2.5
 
 Easiest way to install ``rhodecode`` is to run::
 
@@ -62,7 +51,7 Step by step installation example
 
 ::
 
-  source /var/www/rhodecode-venv/bin/activate
+  source activate /var/www/rhodecode-venv/bin/activate
      
 - Make a folder for rhodecode somewhere on the filesystem for example 
 
@@ -80,8 +69,28 Step by step installation example
 - this will install rhodecode together with pylons
   and all other required python libraries
 
+Requirements for Celery (optional)
+----------------------------------
+
+.. note::
+   Installing message broker and using celery is optional, RhodeCode will
+   work without them perfectly fine.
+
+
+**Message Broker** 
+
+- preferred is `RabbitMq <http://www.rabbitmq.com/>`_
+- possible other is `Redis <http://code.google.com/p/redis/>`_
+
+For installation instructions You can visit: 
+http://ask.github.com/celery/getting-started/index.html
+It's very nice tutorial how to start celery_ with rabbitmq_
+
 
 You can now proceed to :ref:`setup`
+-----------------------------------
+
+
 
 .. _virtualenv: http://pypi.python.org/pypi/virtualenv  
 .. _python: http://www.python.org/

docs/setup.rst

@@ -7,13 +7,20 Setup
 Setting up the application
 --------------------------
 
+First You'll ned to create RhodeCode config file. Run the following command 
+to do this
+
 ::
  
  paster make-config RhodeCode production.ini
 
 - This will create `production.ini` config inside the directory
-  this config contain various settings for rhodecode, e.g port, email settings
-  static files, cache and logging.
+  this config contains various settings for RhodeCode, e.g proxy port, 
+  email settings, usage of static files, cache, celery settings and logging.
+
+
+
+Next we need to create the database.
 
 ::
 
@@ -24,55 +31,136 Setting up the application
   existing ones. RhodeCode will simply add all new found repositories to 
   it's database. Also make sure You specify correct path to repositories.
 - Remember that the given path for mercurial_ repositories must be write 
-  accessible for the application. It's very important since RhodeCode web interface
-  will work even without such an access but, when trying to do a push it'll 
-  eventually fail with permission denied errors. 
-- Run 
+  accessible for the application. It's very important since RhodeCode web 
+  interface will work even without such an access but, when trying to do a 
+  push it'll eventually fail with permission denied errors. 
+
+You are ready to use rhodecode, to run it simply execute
 
 ::
  
  paster serve production.ini
  
-- This command runs the rhodecode server the app should be available at the 
+- This command runs the RhodeCode server the app should be available at the 
   127.0.0.1:5000. This ip and port is configurable via the production.ini 
   file  created in previous step
 - Use admin account you created to login.
 - Default permissions on each repository is read, and owner is admin. So 
-  remember to update these if needed.
+  remember to update these if needed. In the admin panel You can toggle ldap,
+  anonymous, permissions settings. As well as edit more advanced options on 
+  users and repositories
+  
+    
+Setting up Whoosh full text search
+----------------------------------
+
+Index for whoosh can be build starting from version 1.1 using paster command
+passing repo locations to index, as well as Your config file that stores
+whoosh index files locations. There is possible to pass `-f` to the options
+to enable full index rebuild. Without that indexing will run always in in
+incremental mode.
+
+::
+
+ paster make-index production.ini --repo-location=<location for repos> 
+
+for full index rebuild You can use
+
+::
   
-Note
-----
+ paster make-index production.ini -f --repo-location=<location for repos>
+
+- For full text search You can either put crontab entry for
+
+This command can be run even from crontab in order to do periodical 
+index builds and keep Your index always up to date. An example entry might 
+look like this
+
+::
+ 
+ /path/to/python/bin/paster /path/to/rhodecode/production.ini --repo-location=<location for repos> 
+  
+When using incremental(default) mode whoosh will check last modification date 
+of each file and add it to reindex if newer file is available. Also indexing 
+daemon checks for removed files and removes them from index. 
+
+Sometime You might want to rebuild index from scratch. You can do that using 
+the `-f` flag passed to paster command or, in admin panel You can check 
+`build from scratch` flag.
+
+
+Setting up LDAP support
+-----------------------
 
-RhodeCode when running without the celery it's running all it's task in sync
-mode, for first few times when visiting summary page You can notice few
-slow downs, this is due the statistics building it's cache. After all changesets
-are parsed it'll take the stats from cache and run much faster. Each summary
-page display parse at most 250 changesets in order to not stress the cpu, so
-the full stats are going to be loaded after total_number_of_changesets/250
-summary page visits.
+RhodeCode starting from version 1.1 supports ldap authentication. In order
+to use ldap, You have to install python-ldap package. This package is available
+via pypi, so You can install it by running
+
+::
+
+ easy_install python-ldap
+ 
+::
+
+ pip install python-ldap
+
+.. note::
+   python-ldap requires some certain libs on Your system, so before installing 
+   it check that You have at least `openldap`, and `sasl` libraries.
+
+ldap settings are located in admin->ldap section,
+
+Here's a typical ldap setup::
+
+ Enable ldap  = checked                 #controls if ldap access is enabled
+ Host         = host.domain.org         #actual ldap server to connect
+ Port         = 389 or 689 for ldaps    #ldap server ports
+ Enable LDAPS = unchecked               #enable disable ldaps
+ Account      = <account>               #access for ldap server(if required)
+ Password     = <password>              #password for ldap server(if required)
+ Base DN      = uid=%(user)s,CN=users,DC=host,DC=domain,DC=org
+ 
+
+`Account` and `Password` are optional, and used for two-phase ldap 
+authentication so those are credentials to access Your ldap, if it doesn't 
+support anonymous search/user lookups. 
+
+Base DN must have %(user)s template inside, it's a placer where Your uid used
+to login would go, it allows admins to specify not standard schema for uid 
+variable
+
+If all data are entered correctly, and `python-ldap` is properly installed
+Users should be granted to access RhodeCode wit ldap accounts. When 
+logging at the first time an special ldap account is created inside RhodeCode, 
+so You can control over permissions even on ldap users. If such user exists 
+already in RhodeCode database ldap user with the same username would be not 
+able to access RhodeCode.
+
+If You have problems with ldap access and believe You entered correct 
+information check out the RhodeCode logs,any error messages sent from 
+ldap will be saved there.
 
 
     
-Setting up Whoosh
+Setting Up Celery
 -----------------
 
-- For full text search You can either put crontab entry for
-
-::
- 
- python /var/www/rhodecode/<rhodecode_installation_path>/lib/indexers/daemon.py incremental <put_here_path_to_repos>
+Since version 1.1 celery is configured by the rhodecode ini configuration files
+simply set use_celery=true in the ini file then add / change the configuration 
+variables inside the ini file.
   
-When using incremental mode whoosh will check last modification date of each file
-and add it to reindex if newer file is available. Also indexing daemon checks
-for removed files and removes them from index. Sometime You might want to rebuild
-index from scratch, in admin panel You can check `build from scratch` flag
-and in standalone daemon You can pass `full` instead on incremental to build
-remove previous index and build new one.
+Remember that the ini files uses format with '.' not with '_' like celery
+so for example setting `BROKER_HOST` in celery means setting `broker.host` in
+the config file.
+
+In order to make start using celery run::
+ paster celeryd <configfile.ini>
+
 
 Nginx virtual host example
 --------------------------
 
-Sample config for nginx::
+Sample config for nginx using proxy::
 
  server {
     listen          80;
@@ -122,6 +210,16 in production.ini file::
 
 To not have the statics served by the application. And improve speed.
 
+Apache reverse proxy
+--------------------
+Tutorial can be found here
+http://wiki.pylonshq.com/display/pylonscookbook/Apache+as+a+reverse+proxy+for+Pylons
+
+
+Apache's example FCGI config
+----------------------------
+
+TODO !
 
 Other configuration files
 -------------------------
@@ -132,6 +230,29 http://hg.python-works.com/rhodecode/fil
 and also an celeryconfig file can be use from here:
 http://hg.python-works.com/rhodecode/files/tip/celeryconfig.py
 
+Troubleshooting
+---------------
+
+- missing static files ?
+
+ - make sure either to set the `static_files = true` in the .ini file or
+   double check the root path for Your http setup. It should point to 
+   for example:
+   /home/my-virtual-python/lib/python2.6/site-packages/rhodecode/public
+   
+- can't install celery/rabbitmq
+
+ - don't worry RhodeCode works without them too. No extra setup required
+
+- long lasting push timeouts ?
+
+ - make sure You set a longer timeouts in Your proxy/fcgi settings, timeouts
+   are caused by https server and not RhodeCode
+
+- large pushes timeouts ?
+ 
+ - make sure You set a proper max_body_size for the http server
+
 
 
 .. _virtualenv: http://pypi.python.org/pypi/virtualenv

docs/upgrade.rst

@@ -22,7 +22,26 Then make sure You run from the installa
  paster make-config RhodeCode production.ini
  
 This will display any changes made from new version of RhodeCode To your
-current config. And tries to do an automerge.
+current config. And tries to do an automerge. It's always better to do a backup
+of config file and recheck the content after merge.
+
+It's also good to rebuild the whoosh index since after upgrading the whoosh 
+version there could be introduced incompatible index changes.
+
+
+The last step is to upgrade the database. To do this simply run
+
+::
+
+ paster upgrade-db production.ini
+ 
+This will upgrade schema, as well as update some default on the database,
+always recheck the settings of the application, if there are no new options
+that need to be set.
+
+.. note::
+   Always perform a database backup before doing upgrade.
+
 
 
 .. _virtualenv: http://pypi.python.org/pypi/virtualenv  

production.ini

@@ -22,6 +22,7 debug = true
 #smtp_password = 
 #smtp_port = 
 #smtp_use_tls = false
+#smtp_use_ssl = true
 
 [server:main]
 ##nr of threads to spawn
@@ -43,6 +44,35 full_stack = true
 static_files = false
 lang=en
 cache_dir = %(here)s/data
+index_dir = %(here)s/data/index
+cut_off_limit = 256000
+
+####################################
+###        CELERY CONFIG        ####
+####################################
+use_celery = false
+broker.host = localhost
+broker.vhost = rabbitmqhost
+broker.port = 5672
+broker.user = rabbitmq
+broker.password = qweqwe
+
+celery.imports = rhodecode.lib.celerylib.tasks
+
+celery.result.backend = amqp
+celery.result.dburi = amqp://
+celery.result.serialier = json
+
+#celery.send.task.error.emails = true
+#celery.amqp.task.result.expires = 18000
+
+celeryd.concurrency = 2
+#celeryd.log.file = celeryd.log
+celeryd.log.level = debug
+celeryd.max.tasks.per.child = 3
+
+#tasks will never be sent to the queue, but executed locally instead.
+celery.always.eager = false
 
 ####################################
 ###         BEAKER CACHE        ####
@@ -136,6 +166,7 level = INFO
 handlers = console
 qualname = routes.middleware
 # "level = DEBUG" logs the route matched and routing variables.
+propagate = 0
 
 [logger_rhodecode]
 level = DEBUG

rhodecode/__init__.py

@@ -1,8 +1,16
-#!/usr/bin/env python
-# encoding: utf-8
-# RhodeCode, a web based repository management based on pylons
-# Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
-# 
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.__init__
+    ~~~~~~~~~~~~~~~~~~
+
+    RhodeCode, a web based repository management based on pylons
+    versioning implementation: http://semver.org/
+
+    :created_on: Apr 9, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
@@ -17,19 +25,28
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
-"""
-Created on April 9, 2010
-RhodeCode, a web based repository management based on pylons
-versioning implementation: http://semver.org/
-@author: marcink
-"""
+
+
+VERSION = (1, 1, 0)
+__version__ = '.'.join((str(each) for each in VERSION[:4]))
+__dbversion__ = 2 #defines current db version for migrations
 
-VERSION = (1, 0, 2,)
+try:
+    from rhodecode.lib.utils import get_current_revision
+    _rev = get_current_revision()
+except ImportError:
+    #this is needed when doing some setup.py operations
+    _rev = False
 
-__version__ = '.'.join((str(each) for each in VERSION[:4]))
+if len(VERSION) > 3 and _rev:
+    __version__ += ' [rev:%s]' % _rev[0]
 
 def get_version():
-    """
-    Returns shorter version (digit parts only) as string.
-    """
+    """Returns shorter version (digit parts only) as string."""
+
     return '.'.join((str(each) for each in VERSION[:3]))
+
+BACKENDS = {
+    'hg': 'Mercurial repository',
+   #'git': 'Git repository',
+}

rhodecode/config/deployment.ini_tmpl

@@ -1,6 +1,6
 ################################################################################
 ################################################################################
-# rhodecode - Pylons environment configuration                                 #
+# RhodeCode - Pylons environment configuration                                 #
 #                                                                              # 
 # The %(here)s variable will be replaced with the parent directory of this file#
 ################################################################################
@@ -10,7 +10,7 debug = true
 ################################################################################
 ## Uncomment and replace with the address which should receive                ## 
 ## any error reports after application crash                                  ##
-## Additionally those settings will be used by rhodecode mailing system       ##
+## Additionally those settings will be used by RhodeCode mailing system       ##
 ################################################################################
 #email_to = admin@localhost
 #error_email_from = paste_error@localhost
@@ -22,13 +22,14 debug = true
 #smtp_password = 
 #smtp_port = 
 #smtp_use_tls = false
+#smtp_use_ssl = true
 
 [server:main]
 ##nr of threads to spawn
 threadpool_workers = 5
 
 ##max request before thread respawn
-threadpool_max_requests = 2
+threadpool_max_requests = 10
 
 ##option to use threads of process
 use_threadpool = true
@@ -43,7 +44,36 full_stack = true
 static_files = true
 lang=en
 cache_dir = %(here)s/data
+index_dir = %(here)s/data/index
 app_instance_uuid = ${app_instance_uuid}
+cut_off_limit = 256000
+
+####################################
+###        CELERY CONFIG        ####
+####################################
+use_celery = false
+broker.host = localhost
+broker.vhost = rabbitmqhost
+broker.port = 5672
+broker.user = rabbitmq
+broker.password = qweqwe
+
+celery.imports = rhodecode.lib.celerylib.tasks
+
+celery.result.backend = amqp
+celery.result.dburi = amqp://
+celery.result.serialier = json
+
+#celery.send.task.error.emails = true
+#celery.amqp.task.result.expires = 18000
+
+celeryd.concurrency = 2
+#celeryd.log.file = celeryd.log
+celeryd.log.level = debug
+celeryd.max.tasks.per.child = 3
+
+#tasks will never be sent to the queue, but executed locally instead.
+celery.always.eager = false
 
 ####################################
 ###         BEAKER CACHE        ####
@@ -62,7 +92,7 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
 
 beaker.cache.sql_cache_short.type=memory
-beaker.cache.sql_cache_short.expire=5
+beaker.cache.sql_cache_short.expire=10
 
 beaker.cache.sql_cache_med.type=memory
 beaker.cache.sql_cache_med.expire=360
@@ -136,6 +166,7 level = INFO
 handlers = console
 qualname = routes.middleware
 # "level = DEBUG" logs the route matched and routing variables.
+propagate = 0
 
 [logger_rhodecode]
 level = DEBUG

rhodecode/config/environment.py

@@ -6,7 +6,7 from rhodecode.config.routing import mak
 from rhodecode.lib.auth import set_available_permissions, set_base_path
 from rhodecode.lib.utils import repo2db_mapper, make_ui, set_rhodecode_config
 from rhodecode.model import init_model
-from rhodecode.model.hg_model import _get_repos_cached_initial
+from rhodecode.model.scm import ScmModel
 from sqlalchemy import engine_from_config
 import logging
 import os
@@ -52,8 +52,9 def load_environment(global_conf, app_co
     test = os.path.split(config['__file__'])[-1] == 'test.ini'
     if test:
         from rhodecode.lib.utils import create_test_env, create_test_index
-        create_test_env('/tmp', config)
-        create_test_index('/tmp/*', True)
+        from rhodecode.tests import  TESTS_TMP_PATH
+        create_test_env(TESTS_TMP_PATH, config)
+        create_test_index(TESTS_TMP_PATH, True)
         
     #MULTIPLE DB configs
     # Setup the SQLAlchemy database engine
@@ -69,7 +70,8 def load_environment(global_conf, app_co
     #init baseui
     config['pylons.app_globals'].baseui = make_ui('db')
     
-    repo2db_mapper(_get_repos_cached_initial(config['pylons.app_globals'], initial))
+    g = config['pylons.app_globals']
+    repo2db_mapper(ScmModel().repo_scan(g.paths[0][1], g.baseui))
     set_available_permissions(config)
     set_base_path(config)
     set_rhodecode_config(config)

rhodecode/config/middleware.py

@@ -8,8 +8,10 from pylons.middleware import ErrorHandl
 from pylons.wsgiapp import PylonsApp
 from routes.middleware import RoutesMiddleware
 from rhodecode.lib.middleware.simplehg import SimpleHg
+from rhodecode.lib.middleware.simplegit import SimpleGit
 from rhodecode.lib.middleware.https_fixup import HttpsFixup
 from rhodecode.config.environment import load_environment
+from paste.gzipper import make_gzip_middleware
 
 def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
     """Create a Pylons WSGI application and return it
@@ -43,6 +45,7 def make_app(global_conf, full_stack=Tru
     # CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares)
     
     app = SimpleHg(app, config)
+    app = SimpleGit(app, config)
     
     if asbool(full_stack):
         # Handle Python exceptions
@@ -65,6 +68,7 def make_app(global_conf, full_stack=Tru
         # Serve static files
         static_app = StaticURLParser(config['pylons.paths']['static_files'])
         app = Cascade([static_app, app])
+        app = make_gzip_middleware(app, global_conf, compress_level=1)
     
     app.config = config
 

rhodecode/config/routing.py

@@ -35,7 +35,7 def make_map(config):
     #==========================================================================
 
     #MAIN PAGE
-    map.connect('hg_home', '/', controller='hg', action='index')
+    map.connect('home', '/', controller='home', action='index')
     map.connect('bugtracker', "http://bitbucket.org/marcinkuzminski/rhodecode/issues", _static=True)
     map.connect('gpl_license', "http://www.gnu.org/licenses/gpl.html", _static=True)
     #ADMIN REPOSITORY REST ROUTES
@@ -73,13 +73,27 def make_map(config):
         m.connect('delete_repo_user', "/repos_delete_user/{repo_name:.*}",
              action="delete_perm_user", conditions=dict(method=["DELETE"],
                                                         function=check_repo))
-
+        #settings actions
+        m.connect('repo_stats', "/repos_stats/{repo_name:.*}",
+             action="repo_stats", conditions=dict(method=["DELETE"],
+                                                        function=check_repo))
+        m.connect('repo_cache', "/repos_cache/{repo_name:.*}",
+             action="repo_cache", conditions=dict(method=["DELETE"],
+                                                        function=check_repo))
     #ADMIN USER REST ROUTES
     map.resource('user', 'users', controller='admin/users', path_prefix='/_admin')
 
     #ADMIN PERMISSIONS REST ROUTES
     map.resource('permission', 'permissions', controller='admin/permissions', path_prefix='/_admin')
 
+
+    ##ADMIN LDAP SETTINGS
+    map.connect('ldap_settings', '/_admin/ldap', controller='admin/ldap_settings',
+                action='ldap_settings', conditions=dict(method=["POST"]))
+    map.connect('ldap_home', '/_admin/ldap', controller='admin/ldap_settings',)
+
+
+
     #ADMIN SETTINGS REST ROUTES
     with map.submapper(path_prefix='/_admin', controller='admin/settings') as m:
         m.connect("admin_settings", "/settings",
@@ -116,6 +130,14 def make_map(config):
         m.connect('admin_home', '', action='index')#main page
         m.connect('admin_add_repo', '/add_repo/{new_repo:[a-z0-9\. _-]*}',
                   action='add_repo')
+
+
+    #USER JOURNAL
+    map.connect('journal', '/_admin/journal', controller='journal',)
+    map.connect('toggle_following', '/_admin/toggle_following', controller='journal',
+                action='toggle_following', conditions=dict(method=["POST"]))
+
+
     #SEARCH
     map.connect('search', '/_admin/search', controller='search',)
     map.connect('search_repo', '/_admin/search/{search_repo:.*}', controller='search')

rhodecode/controllers/admin/admin.py

@@ -1,8 +1,15
-#!/usr/bin/env python
-# encoding: utf-8
-# admin controller for pylons 
-# Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.controllers.admin.admin
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
+    Controller for Admin panel of Rhodecode
+    
+    :created_on: Apr 7, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
@@ -17,15 +24,10
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
-"""
-Created on April 7, 2010
-admin controller for pylons
-@author: marcink
-"""
+
 import logging
-from pylons import request, response, session, tmpl_context as c
+from pylons import request, tmpl_context as c
 from rhodecode.lib.base import BaseController, render
-from rhodecode.model import meta
 from rhodecode.model.db import UserLog
 from webhelpers.paginate import Page
 from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator

rhodecode/controllers/admin/permissions.py

@@ -1,8 +1,15
-#!/usr/bin/env python
-# encoding: utf-8
-# permissions controller for pylons
-# Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
-#
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.controllers.admin.permissions
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    
+    permissions controller for Rhodecode
+    
+    :created_on: Apr 27, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
@@ -17,11 +24,6
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
-"""
-Created on April 27, 2010
-permissions controller for pylons
-@author: marcink
-"""
 
 from formencode import htmlfill
 from pylons import request, session, tmpl_context as c, url
@@ -29,11 +31,12 from pylons.controllers.util import abor
 from pylons.i18n.translation import _
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator
+from rhodecode.lib.auth_ldap import LdapImportError
 from rhodecode.lib.base import BaseController, render
-from rhodecode.model.db import User, UserLog
-from rhodecode.model.forms import UserForm, DefaultPermissionsForm
-from rhodecode.model.permission_model import PermissionModel
-from rhodecode.model.user_model import UserModel
+from rhodecode.model.forms import LdapSettingsForm, DefaultPermissionsForm
+from rhodecode.model.permission import PermissionModel
+from rhodecode.model.settings import SettingsModel
+from rhodecode.model.user import UserModel
 import formencode
 import logging
 import traceback
@@ -58,7 +61,8 class PermissionsController(BaseControll
                               ('repository.write', _('Write'),),
                               ('repository.admin', _('Admin'),)]
         self.register_choices = [
-            ('hg.register.none', 'disabled'),
+            ('hg.register.none',
+                _('disabled')),
             ('hg.register.manual_activate',
                             _('allowed with manual account activation')),
             ('hg.register.auto_activate',
@@ -99,17 +103,18 class PermissionsController(BaseControll
             form_result = _form.to_python(dict(request.POST))
             form_result.update({'perm_user_name':id})
             permission_model.update(form_result)
-            h.flash(_('Default permissions updated succesfully'),
+            h.flash(_('Default permissions updated successfully'),
                     category='success')
                            
         except formencode.Invalid, errors:
             c.perms_choices = self.perms_choices
             c.register_choices = self.register_choices
             c.create_choices = self.create_choices
+            defaults = errors.value
                     
             return htmlfill.render(
                 render('admin/permissions/permissions.html'),
-                defaults=errors.value,
+                defaults=defaults,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8") 
@@ -143,8 +148,11 class PermissionsController(BaseControll
         c.create_choices = self.create_choices
         
         if id == 'default':
-            defaults = {'_method':'put'}
-            for p in UserModel().get_default().user_perms:
+            default_user = UserModel().get_by_username('default')
+            defaults = {'_method':'put',
+                        'anonymous':default_user.active}
+
+            for p in default_user.user_perms:
                 if p.permission.permission_name.startswith('repository.'):
                     defaults['default_perm'] = p.permission.permission_name 
                 

rhodecode/controllers/admin/repos.py

@@ -1,8 +1,15
-#!/usr/bin/env python
-# encoding: utf-8
-# repos controller for pylons
-# Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
-#
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.controllers.admin.repos
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    
+    Admin controller for RhodeCode
+    
+    :created_on: Apr 7, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
@@ -17,17 +24,18
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
-"""
-Created on April 7, 2010
-admin controller for pylons
-@author: marcink
-"""
+
+import logging
+import traceback
+import formencode
+from operator import itemgetter
 from formencode import htmlfill
-from operator import itemgetter
+
 from paste.httpexceptions import HTTPInternalServerError
 from pylons import request, response, session, tmpl_context as c, url
 from pylons.controllers.util import abort, redirect
 from pylons.i18n.translation import _
+
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator, \
     HasPermissionAnyDecorator
@@ -35,11 +43,9 from rhodecode.lib.base import BaseContr
 from rhodecode.lib.utils import invalidate_cache, action_logger
 from rhodecode.model.db import User
 from rhodecode.model.forms import RepoForm
-from rhodecode.model.hg_model import HgModel
-from rhodecode.model.repo_model import RepoModel
-import formencode
-import logging
-import traceback
+from rhodecode.model.scm import ScmModel
+from rhodecode.model.repo import RepoModel
+
 
 log = logging.getLogger(__name__)
 
@@ -60,7 +66,7 class ReposController(BaseController):
     def index(self, format='html'):
         """GET /repos: All items in the collection"""
         # url('repos')
-        cached_repo_list = HgModel().get_repos()
+        cached_repo_list = ScmModel().get_repos()
         c.repos_list = sorted(cached_repo_list, key=itemgetter('name_sort'))
         return render('admin/repos/repos.html')
     
@@ -74,7 +80,6 class ReposController(BaseController):
         try:
             form_result = _form.to_python(dict(request.POST))
             repo_model.create(form_result, c.rhodecode_user)
-            invalidate_cache('cached_repo_list')
             h.flash(_('created repository %s') % form_result['repo_name'],
                     category='success')
 
@@ -106,7 +111,7 class ReposController(BaseController):
                     % form_result.get('repo_name')
             h.flash(msg, category='error')
         if request.POST.get('user_created'):
-            return redirect(url('hg_home'))    
+            return redirect(url('home'))
         return redirect(url('repos'))
     
     @HasPermissionAllDecorator('hg.admin')
@@ -133,12 +138,29 class ReposController(BaseController):
         try:
             form_result = _form.to_python(dict(request.POST))
             repo_model.update(repo_name, form_result)
-            invalidate_cache('cached_repo_list')
-            h.flash(_('Repository %s updated succesfully' % repo_name),
+            invalidate_cache('get_repo_cached_%s' % repo_name)
+            h.flash(_('Repository %s updated successfully' % repo_name),
                     category='success')
             changed_name = form_result['repo_name']
+            action_logger(self.rhodecode_user, 'admin_updated_repo',
+                              changed_name, '', self.sa)
+
         except formencode.Invalid, errors:
-            c.repo_info = repo_model.get(repo_name)
+            c.repo_info = repo_model.get_by_repo_name(repo_name)
+            if c.repo_info.stats:
+                last_rev = c.repo_info.stats.stat_on_revision
+            else:
+                last_rev = 0
+            c.stats_revision = last_rev
+            r = ScmModel().get(repo_name)
+            c.repo_last_rev = r.revisions[-1] if r.revisions else 0
+
+            if last_rev == 0:
+                c.stats_percentage = 0
+            else:
+                c.stats_percentage = '%.2f' % ((float((last_rev)) /
+                                                c.repo_last_rev) * 100)
+
             c.users_array = repo_model.get_users_js()
             errors.value.update({'user':c.repo_info.user.username})
             return htmlfill.render(
@@ -150,7 +172,7 class ReposController(BaseController):
  
         except Exception:
             log.error(traceback.format_exc())
-            h.flash(_('error occured during update of repository %s') \
+            h.flash(_('error occurred during update of repository %s') \
                     % repo_name, category='error')
             
         return redirect(url('edit_repo', repo_name=changed_name))
@@ -166,7 +188,7 class ReposController(BaseController):
         # url('repo', repo_name=ID)
         
         repo_model = RepoModel()
-        repo = repo_model.get(repo_name)
+        repo = repo_model.get_by_repo_name(repo_name)
         if not repo:
             h.flash(_('%s repository is not mapped to db perhaps' 
                       ' it was moved or renamed  from the filesystem'
@@ -179,7 +201,7 class ReposController(BaseController):
             action_logger(self.rhodecode_user, 'admin_deleted_repo',
                               repo_name, '', self.sa)
             repo_model.delete(repo)            
-            invalidate_cache('cached_repo_list')
+            invalidate_cache('get_repo_cached_%s' % repo_name)
             h.flash(_('deleted repository %s') % repo_name, category='success')
            
         except Exception, e:
@@ -205,6 +227,35 class ReposController(BaseController):
             raise HTTPInternalServerError()
     
     @HasPermissionAllDecorator('hg.admin')    
+    def repo_stats(self, repo_name):
+        """
+        DELETE an existing repository statistics
+        :param repo_name:
+        """
+
+        try:
+            repo_model = RepoModel()
+            repo_model.delete_stats(repo_name)
+        except Exception, e:
+            h.flash(_('An error occured during deletion of repository stats'),
+                    category='error')
+        return redirect(url('edit_repo', repo_name=repo_name))
+
+    @HasPermissionAllDecorator('hg.admin')
+    def repo_cache(self, repo_name):
+        """
+        INVALIDATE exisitings repository cache
+        :param repo_name:
+        """
+
+        try:
+            ScmModel().mark_for_invalidation(repo_name)
+        except Exception, e:
+            h.flash(_('An error occurred during cache invalidation'),
+                    category='error')
+        return redirect(url('edit_repo', repo_name=repo_name))
+
+    @HasPermissionAllDecorator('hg.admin')
     def show(self, repo_name, format='html'):
         """GET /repos/repo_name: Show a specific item"""
         # url('repo', repo_name=ID)
@@ -214,8 +265,10 class ReposController(BaseController):
         """GET /repos/repo_name/edit: Form to edit an existing item"""
         # url('edit_repo', repo_name=ID)
         repo_model = RepoModel()
-        c.repo_info = repo = repo_model.get(repo_name)
-        if not repo:
+        r = ScmModel().get(repo_name)
+        c.repo_info = repo_model.get_by_repo_name(repo_name)
+
+        if c.repo_info is None:
             h.flash(_('%s repository is not mapped to db perhaps' 
                       ' it was created or renamed from the filesystem'
                       ' please run the application again'
@@ -223,7 +276,22 class ReposController(BaseController):
                       category='error')
         
             return redirect(url('repos'))        
-        defaults = c.repo_info.__dict__
+
+        if c.repo_info.stats:
+            last_rev = c.repo_info.stats.stat_on_revision
+        else:
+            last_rev = 0
+        c.stats_revision = last_rev
+
+        c.repo_last_rev = r.revisions[-1] if r.revisions else 0
+
+        if last_rev == 0:
+            c.stats_percentage = 0
+        else:
+            c.stats_percentage = '%.2f' % ((float((last_rev)) /
+                                            c.repo_last_rev) * 100)
+
+        defaults = c.repo_info.get_dict()
         if c.repo_info.user:
             defaults.update({'user':c.repo_info.user.username})
         else:

rhodecode/controllers/admin/settings.py

@@ -1,8 +1,14
-#!/usr/bin/env python
-# encoding: utf-8
-# settings controller for pylons
-# Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
-#
+# -*- coding: utf-8 -*-
+"""
+    package.rhodecode.controllers.admin.settings
+    ~~~~~~~~~~~~~~
+    settings controller for rhodecode admin
+        
+    :created_on: Jul 14, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
@@ -17,11 +23,7
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
-"""
-Created on July 14, 2010
-settings controller for pylons
-@author: marcink
-"""
+
 from formencode import htmlfill
 from pylons import request, session, tmpl_context as c, url, app_globals as g, \
     config
@@ -29,16 +31,18 from pylons.controllers.util import abor
 from pylons.i18n.translation import _
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator, \
-    HasPermissionAnyDecorator
+    HasPermissionAnyDecorator, NotAnonymous
 from rhodecode.lib.base import BaseController, render
+from rhodecode.lib.celerylib import tasks, run_task
 from rhodecode.lib.utils import repo2db_mapper, invalidate_cache, \
-    set_rhodecode_config, get_hg_settings, get_hg_ui_settings, make_ui
-from rhodecode.model.db import User, UserLog, RhodeCodeSettings, RhodeCodeUi
+    set_rhodecode_config
+from rhodecode.model.db import RhodeCodeUi, Repository
 from rhodecode.model.forms import UserForm, ApplicationSettingsForm, \
     ApplicationUiSettingsForm
-from rhodecode.model.hg_model import HgModel
-from rhodecode.model.user_model import UserModel
-from rhodecode.lib.celerylib import tasks, run_task
+from rhodecode.model.scm import ScmModel
+from rhodecode.model.settings import SettingsModel
+from rhodecode.model.user import UserModel
+from sqlalchemy import func
 import formencode
 import logging
 import traceback
@@ -66,8 +70,8 class SettingsController(BaseController)
         """GET /admin/settings: All items in the collection"""
         # url('admin_settings')
 
-        defaults = get_hg_settings()
-        defaults.update(get_hg_ui_settings())
+        defaults = SettingsModel().get_app_settings()
+        defaults.update(self.get_hg_ui_settings())
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=defaults,
@@ -98,13 +102,16 class SettingsController(BaseController)
             rm_obsolete = request.POST.get('destroy', False)
             log.debug('Rescanning directories with destroy=%s', rm_obsolete)
 
-            initial = HgModel.repo_scan(g.paths[0][0], g.paths[0][1], g.baseui)
+            initial = ScmModel().repo_scan(g.paths[0][1], g.baseui)
+            for repo_name in initial.keys():
+                invalidate_cache('get_repo_cached_%s' % repo_name)
+
             repo2db_mapper(initial, rm_obsolete)
-            invalidate_cache('cached_repo_list')
+
             h.flash(_('Repositories successfully rescanned'), category='success')            
         
         if setting_id == 'whoosh':
-            repo_location = get_hg_ui_settings()['paths_root_path']
+            repo_location = self.get_hg_ui_settings()['paths_root_path']
             full_index = request.POST.get('full_index', False)
             task = run_task(tasks.whoosh_index, repo_location, full_index)
             
@@ -114,14 +121,12 class SettingsController(BaseController)
             application_form = ApplicationSettingsForm()()
             try:
                 form_result = application_form.to_python(dict(request.POST))
-            
+                settings_model = SettingsModel()
                 try:
-                    hgsettings1 = self.sa.query(RhodeCodeSettings)\
-                    .filter(RhodeCodeSettings.app_settings_name == 'title').one()
+                    hgsettings1 = settings_model.get('title')
                     hgsettings1.app_settings_value = form_result['rhodecode_title'] 
                     
-                    hgsettings2 = self.sa.query(RhodeCodeSettings)\
-                    .filter(RhodeCodeSettings.app_settings_name == 'realm').one()
+                    hgsettings2 = settings_model.get('realm')
                     hgsettings2.app_settings_value = form_result['rhodecode_realm'] 
                     
                     
@@ -173,13 +178,21 class SettingsController(BaseController)
                     .filter(RhodeCodeUi.ui_key == 'changegroup.repo_size').one()
                     hgsettings4.ui_active = bool(form_result['hooks_changegroup_repo_size'])                                          
                     
+                    hgsettings5 = self.sa.query(RhodeCodeUi)\
+                    .filter(RhodeCodeUi.ui_key == 'pretxnchangegroup.push_logger').one()
+                    hgsettings5.ui_active = bool(form_result['hooks_pretxnchangegroup_push_logger'])
                     
+                    hgsettings6 = self.sa.query(RhodeCodeUi)\
+                    .filter(RhodeCodeUi.ui_key == 'preoutgoing.pull_logger').one()
+                    hgsettings6.ui_active = bool(form_result['hooks_preoutgoing_pull_logger'])
                     
                     
                     self.sa.add(hgsettings1)
                     self.sa.add(hgsettings2)
                     self.sa.add(hgsettings3)
                     self.sa.add(hgsettings4)
+                    self.sa.add(hgsettings5)
+                    self.sa.add(hgsettings6)
                     self.sa.commit()
                     
                     h.flash(_('Updated mercurial settings'),
@@ -225,24 +238,27 class SettingsController(BaseController)
         """GET /admin/settings/setting_id/edit: Form to edit an existing item"""
         # url('admin_edit_setting', setting_id=ID)
 
-
+    @NotAnonymous()
     def my_account(self):
         """
         GET /_admin/my_account Displays info about my account 
         """
         # url('admin_settings_my_account')
-        c.user = self.sa.query(User).get(c.rhodecode_user.user_id)
-        c.user_repos = []
-        for repo in c.cached_repo_list.values():
-            if repo.dbrepo.user.username == c.user.username:
-                c.user_repos.append(repo)
+
+        c.user = UserModel().get(c.rhodecode_user.user_id, cache=False)
+        all_repos = self.sa.query(Repository)\
+            .filter(Repository.user_id == c.user.user_id)\
+            .order_by(func.lower(Repository.repo_name))\
+            .all()
+
+        c.user_repos = ScmModel().get_repos(all_repos)
                 
         if c.user.username == 'default':
             h.flash(_("You can't edit this user since it's" 
               " crucial for entire application"), category='warning')
             return redirect(url('users'))
         
-        defaults = c.user.__dict__
+        defaults = c.user.get_dict()
         return htmlfill.render(
             render('admin/users/user_edit_my_account.html'),
             defaults=defaults,
@@ -266,15 +282,18 class SettingsController(BaseController)
         try:
             form_result = _form.to_python(dict(request.POST))
             user_model.update_my_account(uid, form_result)
-            h.flash(_('Your account was updated succesfully'),
+            h.flash(_('Your account was updated successfully'),
                     category='success')
                            
         except formencode.Invalid, errors:
-            c.user = self.sa.query(User).get(c.rhodecode_user.user_id)
-            c.user_repos = []
-            for repo in c.cached_repo_list.values():
-                if repo.dbrepo.user.username == c.user.username:
-                    c.user_repos.append(repo)            
+            c.user = user_model.get(c.rhodecode_user.user_id, cache=False)
+            c.user = UserModel().get(c.rhodecode_user.user_id, cache=False)
+            all_repos = self.sa.query(Repository)\
+                .filter(Repository.user_id == c.user.user_id)\
+                .order_by(func.lower(Repository.repo_name))\
+                .all()
+            c.user_repos = ScmModel().get_repos(all_repos)
+
             return htmlfill.render(
                 render('admin/users/user_edit_my_account.html'),
                 defaults=errors.value,
@@ -283,11 +302,12 class SettingsController(BaseController)
                 encoding="UTF-8")
         except Exception:
             log.error(traceback.format_exc())
-            h.flash(_('error occured during update of user %s') \
+            h.flash(_('error occurred during update of user %s') \
                     % form_result.get('username'), category='error')
                     
         return redirect(url('my_account'))
     
+    @NotAnonymous()
     @HasPermissionAnyDecorator('hg.admin', 'hg.create.repository')
     def create_repository(self):
         """GET /_admin/create_repository: Form to create a new item"""
@@ -296,3 +316,24 class SettingsController(BaseController)
 
         return render('admin/repos/repo_add_create_repository.html')
         
+    def get_hg_ui_settings(self):
+        ret = self.sa.query(RhodeCodeUi).all()
+
+        if not ret:
+            raise Exception('Could not get application ui settings !')
+        settings = {}
+        for each in ret:
+            k = each.ui_key
+            v = each.ui_value
+            if k == '/':
+                k = 'root_path'
+
+            if k.find('.') != -1:
+                k = k.replace('.', '_')
+
+            if each.ui_section == 'hooks':
+                v = each.ui_active
+
+            settings[each.ui_section + '_' + k] = v
+
+        return settings

rhodecode/controllers/admin/users.py

@@ -1,8 +1,15
-#!/usr/bin/env python
-# encoding: utf-8
-# users controller for pylons
-# Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
-# 
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.controllers.admin.users
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Users crud controller for pylons
+    
+    :created_on: Apr 4, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
@@ -17,26 +24,24
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
-from rhodecode.lib.utils import action_logger
-"""
-Created on April 4, 2010
-users controller for pylons
-@author: marcink
-"""
+
+import logging
+import traceback
+import formencode
 
 from formencode import htmlfill
 from pylons import request, session, tmpl_context as c, url
 from pylons.controllers.util import abort, redirect
 from pylons.i18n.translation import _
+
+from rhodecode.lib.exceptions import *
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator
 from rhodecode.lib.base import BaseController, render
-from rhodecode.model.db import User, UserLog
+
+from rhodecode.model.db import User
 from rhodecode.model.forms import UserForm
-from rhodecode.model.user_model import UserModel, DefaultUserException
-import formencode
-import logging
-import traceback
+from rhodecode.model.user import UserModel
 
 log = logging.getLogger(__name__)
 
@@ -100,7 +105,7 class UsersController(BaseController):
         #           method='put')
         # url('user', id=ID)
         user_model = UserModel()
-        c.user = user_model.get_user(id)
+        c.user = user_model.get(id)
         
         _form = UserForm(edit=True, old_data={'user_id':id,
                                               'email':c.user.email})()
@@ -119,7 +124,7 class UsersController(BaseController):
                 encoding="UTF-8") 
         except Exception:
             log.error(traceback.format_exc())
-            h.flash(_('error occured during update of user %s') \
+            h.flash(_('error occurred during update of user %s') \
                     % form_result.get('username'), category='error')
             
         return redirect(url('users'))
@@ -136,7 +141,7 class UsersController(BaseController):
         try:
             user_model.delete(id)
             h.flash(_('sucessfully deleted user'), category='success')
-        except DefaultUserException, e:
+        except (UserOwnsReposException, DefaultUserException), e:
             h.flash(str(e), category='warning')
         except Exception:
             h.flash(_('An error occured during deletion of user'),
@@ -155,11 +160,10 class UsersController(BaseController):
         if not c.user:
             return redirect(url('users'))
         if c.user.username == 'default':
-            h.flash(_("You can't edit this user since it's" 
-              " crucial for entire application"), category='warning')
+            h.flash(_("You can't edit this user"), category='warning')
             return redirect(url('users'))
         
-        defaults = c.user.__dict__
+        defaults = c.user.get_dict()
         return htmlfill.render(
             render('admin/users/user_edit.html'),
             defaults=defaults,

rhodecode/controllers/branches.py

@@ -26,7 +26,7 from pylons import tmpl_context as c
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseController, render
 from rhodecode.lib.utils import OrderedDict
-from rhodecode.model.hg_model import HgModel
+from rhodecode.model.scm import ScmModel
 import logging
 log = logging.getLogger(__name__)
 
@@ -38,7 +38,7 class BranchesController(BaseController)
         super(BranchesController, self).__before__()
     
     def index(self):
-        hg_model = HgModel()
+        hg_model = ScmModel()
         c.repo_info = hg_model.get_repo(c.repo_name)
         c.repo_branches = OrderedDict()
         for name, hash_ in c.repo_info.branches.items():

rhodecode/controllers/changelog.py

@@ -32,7 +32,7 from mercurial.graphmod import colored,
 from pylons import request, session, tmpl_context as c
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseController, render
-from rhodecode.model.hg_model import HgModel
+from rhodecode.model.scm import ScmModel
 from webhelpers.paginate import Page
 import logging
 log = logging.getLogger(__name__)
@@ -60,7 +60,7 class ChangelogController(BaseController
         else:
             c.size = int(session.get('changelog_size', default))
 
-        changesets = HgModel().get_repo(c.repo_name)
+        changesets = ScmModel().get_repo(c.repo_name)
             
         p = int(request.params.get('page', 1))
         c.total_cs = len(changesets)
@@ -74,10 +74,14 class ChangelogController(BaseController
 
     def _graph(self, repo, size, p):
         revcount = size
-        if not repo.revisions:return json.dumps([]), 0
+        if not repo.revisions or repo.alias == 'git':
+            c.jsdata = json.dumps([])
+            return
         
         max_rev = repo.revisions[-1]
+
         offset = 1 if p == 1 else  ((p - 1) * revcount + 1)
+
         rev_start = repo.revisions[(-1 * offset)]
         
         revcount = min(max_rev, revcount)

rhodecode/controllers/changeset.py

@@ -1,7 +1,15
-#!/usr/bin/env python
-# encoding: utf-8
-# changeset controller for pylons
-# Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.controllers.changeset
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    changeset controller for pylons
+    
+    :created_on: Apr 25, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
@@ -16,23 +24,22
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
-from rhodecode.lib.utils import EmptyChangeset
-"""
-Created on April 25, 2010
-changeset controller for pylons
-@author: marcink
-"""
+import logging
+import traceback
+
 from pylons import tmpl_context as c, url, request, response
 from pylons.i18n.translation import _
 from pylons.controllers.util import redirect
+
+import rhodecode.lib.helpers as h
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseController, render
-from rhodecode.model.hg_model import HgModel
+from rhodecode.lib.utils import EmptyChangeset
+from rhodecode.model.scm import ScmModel
+
 from vcs.exceptions import RepositoryError, ChangesetError
 from vcs.nodes import FileNode
 from vcs.utils import diffs as differ
-import logging
-import traceback
 
 log = logging.getLogger(__name__)
 
@@ -45,8 +52,7 class ChangesetController(BaseController
         super(ChangesetController, self).__before__()
         
     def index(self, revision):
-        hg_model = HgModel()
-        cut_off_limit = 1024 * 250
+        hg_model = ScmModel()
         
         def wrap_to_table(str):
             
@@ -59,9 +65,10 class ChangesetController(BaseController
             
         try:
             c.changeset = hg_model.get_repo(c.repo_name).get_changeset(revision)
-        except RepositoryError:
+        except RepositoryError, e:
             log.error(traceback.format_exc())
-            return redirect(url('hg_home'))
+            h.flash(str(e), category='warning')
+            return redirect(url('home'))
         else:
             try:
                 c.changeset_old = c.changeset.parents[0]
@@ -80,7 +87,7 class ChangesetController(BaseController
                     diff = wrap_to_table(_('binary file'))
                 else:
                     c.sum_added += node.size
-                    if c.sum_added < cut_off_limit:
+                    if c.sum_added < self.cut_off_limit:
                         f_udiff = differ.get_udiff(filenode_old, node)
                         diff = differ.DiffProcessor(f_udiff).as_html()
                                                     
@@ -89,7 +96,7 class ChangesetController(BaseController
                                             ' off, see raw changeset instead'))
                         
                 cs1 = None
-                cs2 = node.last_changeset.short_id                                        
+                cs2 = node.last_changeset.raw_id
                 c.changes.append(('added', node, diff, cs1, cs2))
             
             #===================================================================
@@ -106,7 +113,7 class ChangesetController(BaseController
                     diff = wrap_to_table(_('binary file'))
                 else:
                     
-                    if c.sum_removed < cut_off_limit:
+                    if c.sum_removed < self.cut_off_limit:
                         f_udiff = differ.get_udiff(filenode_old, node)
                         diff = differ.DiffProcessor(f_udiff).as_html()
                         if diff:
@@ -116,8 +123,8 class ChangesetController(BaseController
                                             ' off, see raw changeset instead'))
                 
                 
-                cs1 = filenode_old.last_changeset.short_id
-                cs2 = node.last_changeset.short_id                    
+                cs1 = filenode_old.last_changeset.raw_id
+                cs2 = node.last_changeset.raw_id
                 c.changes.append(('changed', node, diff, cs1, cs2))
                 
             #===================================================================
@@ -130,13 +137,15 class ChangesetController(BaseController
 
     def raw_changeset(self, revision):
         
-        hg_model = HgModel()
+        hg_model = ScmModel()
         method = request.GET.get('diff', 'show')
         try:
-            c.changeset = hg_model.get_repo(c.repo_name).get_changeset(revision)
+            r = hg_model.get_repo(c.repo_name)
+            c.scm_type = r.alias
+            c.changeset = r.get_changeset(revision)
         except RepositoryError:
             log.error(traceback.format_exc())
-            return redirect(url('hg_home'))
+            return redirect(url('home'))
         else:
             try:
                 c.changeset_old = c.changeset.parents[0]
@@ -147,13 +156,13 class ChangesetController(BaseController
             for node in c.changeset.added:
                 filenode_old = FileNode(node.path, '')
                 if filenode_old.is_binary or node.is_binary:
-                    diff = _('binary file')
+                    diff = _('binary file') + '\n'
                 else:    
                     f_udiff = differ.get_udiff(filenode_old, node)
                     diff = differ.DiffProcessor(f_udiff).raw_diff()
 
                 cs1 = None
-                cs2 = node.last_changeset.short_id                                        
+                cs2 = node.last_changeset.raw_id
                 c.changes.append(('added', node, diff, cs1, cs2))
                 
             for node in c.changeset.changed:
@@ -164,13 +173,15 class ChangesetController(BaseController
                     f_udiff = differ.get_udiff(filenode_old, node)
                     diff = differ.DiffProcessor(f_udiff).raw_diff()
 
-                cs1 = filenode_old.last_changeset.short_id
-                cs2 = node.last_changeset.short_id                    
+                cs1 = filenode_old.last_changeset.raw_id
+                cs2 = node.last_changeset.raw_id
                 c.changes.append(('changed', node, diff, cs1, cs2))      
         
         response.content_type = 'text/plain'
+
         if method == 'download':
             response.content_disposition = 'attachment; filename=%s.patch' % revision 
+
         parent = True if len(c.changeset.parents) > 0 else False
         c.parent_tmpl = 'Parent  %s' % c.changeset.parents[0].raw_id if parent else ''
     

rhodecode/controllers/error.py

@@ -1,27 +1,52
-import logging
+# -*- coding: utf-8 -*-
+"""
+    package.rhodecode.controllers.error
+    ~~~~~~~~~~~~~~
+
+    RhodeCode error controller
+    
+    :created_on: Dec 8, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your opinion) any later version of the license.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+import os
 import cgi
-import os
+import logging
 import paste.fileapp
-from pylons import tmpl_context as c, app_globals as g, request, config
-from pylons.controllers.util import forward
+
+from pylons import tmpl_context as c, request
 from pylons.i18n.translation import _
+from pylons.middleware import  media_path
+
 from rhodecode.lib.base import BaseController, render
-from pylons.middleware import  media_path
-from rhodecode.lib.utils import check_repo
-import rhodecode.lib.helpers as h
-from rhodecode import __version__
+
 log = logging.getLogger(__name__)
 
 class ErrorController(BaseController):
-    """
-    Generates error documents as and when they are required.
+    """Generates error documents as and when they are required.
 
     The ErrorDocuments middleware forwards to ErrorController when error
     related status codes are returned from the application.
 
-    This behaviour can be altered by changing the parameters to the
+    This behavior can be altered by changing the parameters to the
     ErrorDocuments middleware in your config/middleware.py file.
     """
+
     def __before__(self):
         pass#disable all base actions since we don't need them here
         
@@ -74,7 +99,7 class ErrorController(BaseController):
         if code == 400:
             return _('The request could not be understood by the server due to malformed syntax.')
         if code == 401:
-            return _('Unathorized access to resource')
+            return _('Unauthorized access to resource')
         if code == 403:
             return _("You don't have permission to view this page")
         if code == 404:

rhodecode/controllers/feed.py

@@ -24,7 +24,7 feed controller for pylons
 """
 from pylons import tmpl_context as c, url, response
 from rhodecode.lib.base import BaseController, render
-from rhodecode.model.hg_model import HgModel
+from rhodecode.model.scm import ScmModel
 from webhelpers.feedgenerator import Atom1Feed, Rss201rev2Feed
 import logging
 log = logging.getLogger(__name__)
@@ -49,12 +49,12 class FeedController(BaseController):
                          language=self.language,
                          ttl=self.ttl)
         
-        changesets = HgModel().get_repo(repo_name)
+        changesets = ScmModel().get_repo(repo_name)
 
         for cs in changesets[:self.feed_nr]:
             feed.add_item(title=cs.message,
                           link=url('changeset_home', repo_name=repo_name,
-                                   revision=cs.short_id, qualified=True),
+                                   revision=cs.raw_id, qualified=True),
                                    description=str(cs.date))
         
         response.content_type = feed.mime_type
@@ -69,11 +69,11 class FeedController(BaseController):
                          language=self.language,
                          ttl=self.ttl)
         
-        changesets = HgModel().get_repo(repo_name)
+        changesets = ScmModel().get_repo(repo_name)
         for cs in changesets[:self.feed_nr]:
             feed.add_item(title=cs.message,
                           link=url('changeset_home', repo_name=repo_name,
-                                   revision=cs.short_id, qualified=True),
+                                   revision=cs.raw_id, qualified=True),
                           description=str(cs.date))
             
         response.content_type = feed.mime_type

rhodecode/controllers/files.py

@@ -1,8 +1,15
-#!/usr/bin/env python
-# encoding: utf-8
-# files controller for pylons
-# Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.controllers.files
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+    Files controller for RhodeCode
+    
+    :created_on: Apr 21, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
@@ -17,25 +24,24
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
-"""
-Created on April 21, 2010
-files controller for pylons
-@author: marcink
-"""
+import tempfile
+import logging
+import rhodecode.lib.helpers as h
+
 from mercurial import archival
+
 from pylons import request, response, session, tmpl_context as c, url
 from pylons.i18n.translation import _
 from pylons.controllers.util import redirect
+
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseController, render
 from rhodecode.lib.utils import EmptyChangeset
-from rhodecode.model.hg_model import HgModel
+from rhodecode.model.scm import ScmModel
+
 from vcs.exceptions import RepositoryError, ChangesetError
 from vcs.nodes import FileNode
 from vcs.utils import diffs as differ
-import logging
-import rhodecode.lib.helpers as h
-import tempfile
         
 log = logging.getLogger(__name__)
 
@@ -46,11 +52,11 class FilesController(BaseController):
                                    'repository.admin')       
     def __before__(self):
         super(FilesController, self).__before__()
-        c.file_size_limit = 250 * 1024 #limit of file size to display
+        c.cut_off_limit = self.cut_off_limit
 
     def index(self, repo_name, revision, f_path):
-        hg_model = HgModel()
-        c.repo = repo = hg_model.get_repo(c.repo_name)
+        hg_model = ScmModel()
+        c.repo = hg_model.get_repo(c.repo_name)
         revision = request.POST.get('at_rev', None) or revision
         
         def get_next_rev(cur):
@@ -68,29 +74,33 class FilesController(BaseController):
      
         
         try:
-            cur_rev = repo.get_changeset(revision).revision
-            prev_rev = repo.get_changeset(get_prev_rev(cur_rev)).short_id
-            next_rev = repo.get_changeset(get_next_rev(cur_rev)).short_id
+            c.changeset = c.repo.get_changeset(revision)
+            cur_rev = c.changeset.revision
+            prev_rev = c.repo.get_changeset(get_prev_rev(cur_rev)).raw_id
+            next_rev = c.repo.get_changeset(get_next_rev(cur_rev)).raw_id
                     
             c.url_prev = url('files_home', repo_name=c.repo_name,
                              revision=prev_rev, f_path=f_path) 
             c.url_next = url('files_home', repo_name=c.repo_name,
                              revision=next_rev, f_path=f_path)   
                     
-            c.changeset = repo.get_changeset(revision)
-                        
-            c.cur_rev = c.changeset.short_id
-            c.rev_nr = c.changeset.revision
+            try:
             c.files_list = c.changeset.get_node(f_path)
-            c.file_history = self._get_history(repo, c.files_list, f_path)
+                c.file_history = self._get_history(c.repo, c.files_list, f_path)
+            except RepositoryError, e:
+                h.flash(str(e), category='warning')
+                redirect(h.url('files_home', repo_name=repo_name, revision=revision))
             
-        except (RepositoryError, ChangesetError):
-            c.files_list = None
+        except RepositoryError, e:
+            h.flash(str(e), category='warning')
+            redirect(h.url('files_home', repo_name=repo_name, revision='tip'))
+
+
         
         return render('files/files.html')
 
     def rawfile(self, repo_name, revision, f_path):
-        hg_model = HgModel()
+        hg_model = ScmModel()
         c.repo = hg_model.get_repo(c.repo_name)
         file_node = c.repo.get_changeset(revision).get_node(f_path)
         response.content_type = file_node.mimetype
@@ -99,7 +109,7 class FilesController(BaseController):
         return file_node.content
 
     def raw(self, repo_name, revision, f_path):
-        hg_model = HgModel()
+        hg_model = ScmModel()
         c.repo = hg_model.get_repo(c.repo_name)
         file_node = c.repo.get_changeset(revision).get_node(f_path)
         response.content_type = 'text/plain'
@@ -107,13 +117,18 class FilesController(BaseController):
         return file_node.content
     
     def annotate(self, repo_name, revision, f_path):
-        hg_model = HgModel()
+        hg_model = ScmModel()
         c.repo = hg_model.get_repo(c.repo_name)
-        cs = c.repo.get_changeset(revision)
-        c.file = cs.get_node(f_path)
-        c.file_msg = cs.get_file_message(f_path)
-        c.cur_rev = cs.short_id
-        c.rev_nr = cs.revision        
+
+        try:
+            c.cs = c.repo.get_changeset(revision)
+            c.file = c.cs.get_node(f_path)
+        except RepositoryError, e:
+            h.flash(str(e), category='warning')
+            redirect(h.url('files_home', repo_name=repo_name, revision=revision))
+
+        c.file_history = self._get_history(c.repo, c.file, f_path)
+
         c.f_path = f_path
 
         return render('files/files_annotate.html')
@@ -137,7 +152,7 class FilesController(BaseController):
                 yield data        
             
         archive = tempfile.TemporaryFile()
-        repo = HgModel().get_repo(repo_name).repo
+        repo = ScmModel().get_repo(repo_name).repo
         fname = '%s-%s%s' % (repo_name, revision, fileformat)
         archival.archive(repo, archive, revision, archive_specs[fileformat][1],
                          prefix='%s-%s' % (repo_name, revision))
@@ -147,7 +162,7 class FilesController(BaseController):
         return read_in_chunks(archive)
     
     def diff(self, repo_name, f_path):
-        hg_model = HgModel()
+        hg_model = ScmModel()
         diff1 = request.GET.get('diff1')
         diff2 = request.GET.get('diff2')
         c.action = request.GET.get('diff')
@@ -173,9 +188,6 class FilesController(BaseController):
             return redirect(url('files_home',
                                 repo_name=c.repo_name, f_path=f_path))
 
-        c.diff1 = 'r%s:%s' % (c.changeset_1.revision, c.changeset_1.short_id)
-        c.diff2 = 'r%s:%s' % (c.changeset_2.revision, c.changeset_2.short_id)
-        
         f_udiff = differ.get_udiff(node1, node2)
         diff = differ.DiffProcessor(f_udiff)
                                 
@@ -187,15 +199,17 class FilesController(BaseController):
             return diff.raw_diff()
         
         elif c.action == 'raw':
-            c.cur_diff = '<pre class="raw">%s</pre>' % h.escape(diff.raw_diff())
+            response.content_type = 'text/plain'
+            return diff.raw_diff()
+
         elif c.action == 'diff':
-            if node1.size > c.file_size_limit or node2.size > c.file_size_limit:
+            if node1.size > self.cut_off_limit or node2.size > self.cut_off_limit:
                 c.cur_diff = _('Diff is to big to display')
             else:
                 c.cur_diff = diff.as_html()
         else:
             #default option
-            if node1.size > c.file_size_limit or node2.size > c.file_size_limit:
+            if node1.size > self.cut_off_limit or node2.size > self.cut_off_limit:
                 c.cur_diff = _('Diff is to big to display')
             else:
                 c.cur_diff = diff.as_html()
@@ -209,7 +223,31 class FilesController(BaseController):
             return []
         changesets = node.history
         hist_l = []
+
+        changesets_group = ([], _("Changesets"))
+        branches_group = ([], _("Branches"))
+        tags_group = ([], _("Tags"))
+
         for chs in changesets:
             n_desc = 'r%s:%s' % (chs.revision, chs.short_id)
-            hist_l.append((chs.short_id, n_desc,))
+            changesets_group[0].append((chs.raw_id, n_desc,))
+
+        hist_l.append(changesets_group)
+
+        for name, chs in c.repository_branches.items():
+            #chs = chs.split(':')[-1]
+            branches_group[0].append((chs, name),)
+        hist_l.append(branches_group)
+
+        for name, chs in c.repository_tags.items():
+            #chs = chs.split(':')[-1]
+            tags_group[0].append((chs, name),)
+        hist_l.append(tags_group)
+
         return hist_l
+
+#                [
+#                 ([("u1", "User1"), ("u2", "User2")], "Users"),
+#                 ([("g1", "Group1"), ("g2", "Group2")], "Groups")
+#                 ]
+

rhodecode/controllers/home.py

@@ -26,15 +26,15 from operator import itemgetter
 from pylons import tmpl_context as c, request
 from rhodecode.lib.auth import LoginRequired
 from rhodecode.lib.base import BaseController, render
-from rhodecode.model.hg_model import HgModel
+from rhodecode.model.scm import ScmModel
 import logging
 log = logging.getLogger(__name__)
 
-class HgController(BaseController):
+class HomeController(BaseController):
 
     @LoginRequired()
     def __before__(self):
-        super(HgController, self).__before__()
+        super(HomeController, self).__before__()
         
     def index(self):
         sortables = ['name', 'description', 'last_change', 'tip', 'contact']
@@ -47,7 +47,7 class HgController(BaseController):
         else:
             c.sort_by = current_sort
         c.sort_slug = current_sort_slug
-        cached_repo_list = HgModel().get_repos()
+        cached_repo_list = ScmModel().get_repos()
         
         sort_key = current_sort_slug + '_sort'
         if c.sort_by.startswith('-'):

rhodecode/controllers/login.py

@@ -31,7 +31,7 from rhodecode.lib.base import BaseContr
 import rhodecode.lib.helpers as h 
 from pylons.i18n.translation import _
 from rhodecode.model.forms import LoginForm, RegisterForm, PasswordResetForm
-from rhodecode.model.user_model import UserModel
+from rhodecode.model.user import UserModel
 import formencode
 import logging
 
@@ -46,8 +46,10 class LoginController(BaseController):
         #redirect if already logged in
         c.came_from = request.GET.get('came_from', None)
         
-        if c.rhodecode_user.is_authenticated:
-            return redirect(url('hg_home'))
+        if c.rhodecode_user.is_authenticated \
+                            and c.rhodecode_user.username != 'default':
+
+            return redirect(url('home'))
         
         if request.POST:
             #import Login Form validator class
@@ -55,7 +57,7 class LoginController(BaseController):
             try:
                 c.form_result = login_form.to_python(dict(request.POST))
                 username = c.form_result['username']
-                user = UserModel().get_user_by_name(username)
+                user = UserModel().get_by_username(username, case_insensitive=True)
                 auth_user = AuthUser()
                 auth_user.username = user.username
                 auth_user.is_authenticated = True
@@ -72,7 +74,7 class LoginController(BaseController):
                 if c.came_from:
                     return redirect(c.came_from)
                 else:
-                    return redirect(url('hg_home'))
+                    return redirect(url('home'))
                                
             except formencode.Invalid, errors:
                 return htmlfill.render(
@@ -89,7 +91,7 class LoginController(BaseController):
     def register(self):
         user_model = UserModel()
         c.auto_active = False
-        for perm in user_model.get_default().user_perms:
+        for perm in user_model.get_by_username('default', cache=False).user_perms:
             if perm.permission.permission_name == 'hg.register.auto_activate':
                 c.auto_active = True
                 break
@@ -141,4 +143,4 class LoginController(BaseController):
         session['rhodecode_user'] = AuthUser()
         session.save()
         log.info('Logging out and setting user as Empty')
-        redirect(url('hg_home'))
+        redirect(url('home'))

rhodecode/controllers/search.py

@@ -22,11 +22,11 Created on Aug 7, 2010
 search controller for pylons
 @author: marcink
 """
-from pylons import request, response, session, tmpl_context as c, url
+from pylons import request, response, config, session, tmpl_context as c, url
 from pylons.controllers.util import abort, redirect
 from rhodecode.lib.auth import LoginRequired
 from rhodecode.lib.base import BaseController, render
-from rhodecode.lib.indexers import IDX_LOCATION, SCHEMA, IDX_NAME, ResultWrapper
+from rhodecode.lib.indexers import SCHEMA, IDX_NAME, ResultWrapper
 from webhelpers.paginate import Page
 from webhelpers.util import update_params
 from pylons.i18n.translation import _
@@ -64,7 +64,8 class SearchController(BaseController):
             p = int(request.params.get('page', 1))
             highlight_items = set()
             try:
-                idx = open_dir(IDX_LOCATION, indexname=IDX_NAME)
+                idx = open_dir(config['app_conf']['index_dir']
+                               , indexname=IDX_NAME)
                 searcher = idx.searcher()
 
                 qp = QueryParser(search_type, schema=SCHEMA)

rhodecode/controllers/settings.py

@@ -30,7 +30,7 from rhodecode.lib.auth import LoginRequ
 from rhodecode.lib.base import BaseController, render
 from rhodecode.lib.utils import invalidate_cache, action_logger
 from rhodecode.model.forms import RepoSettingsForm, RepoForkForm
-from rhodecode.model.repo_model import RepoModel
+from rhodecode.model.repo import RepoModel
 import formencode
 import logging
 import rhodecode.lib.helpers as h
@@ -47,7 +47,7 class SettingsController(BaseController)
         
     def index(self, repo_name):
         repo_model = RepoModel()
-        c.repo_info = repo = repo_model.get(repo_name)
+        c.repo_info = repo = repo_model.get_by_repo_name(repo_name)
         if not repo:
             h.flash(_('%s repository is not mapped to db perhaps' 
                       ' it was created or renamed from the filesystem'
@@ -55,8 +55,8 class SettingsController(BaseController)
                       ' in order to rescan repositories') % repo_name,
                       category='error')
         
-            return redirect(url('hg_home'))        
-        defaults = c.repo_info.__dict__
+            return redirect(url('home'))
+        defaults = c.repo_info.get_dict()
         defaults.update({'user':c.repo_info.user.username})
         c.users_array = repo_model.get_users_js()
         
@@ -78,12 +78,14 class SettingsController(BaseController)
         try:
             form_result = _form.to_python(dict(request.POST))
             repo_model.update(repo_name, form_result)
-            invalidate_cache('cached_repo_list')
+            invalidate_cache('get_repo_cached_%s' % repo_name)
             h.flash(_('Repository %s updated successfully' % repo_name),
                     category='success')
             changed_name = form_result['repo_name']               
+            action_logger(self.rhodecode_user, 'user_updated_repo',
+                              changed_name, '', self.sa)
         except formencode.Invalid, errors:
-            c.repo_info = repo_model.get(repo_name)
+            c.repo_info = repo_model.get_by_repo_name(repo_name)
             c.users_array = repo_model.get_users_js()
             errors.value.update({'user':c.repo_info.user.username})
             return htmlfill.render(
@@ -94,7 +96,7 class SettingsController(BaseController)
                 encoding="UTF-8") 
         except Exception:
             log.error(traceback.format_exc())
-            h.flash(_('error occured during update of repository %s') \
+            h.flash(_('error occurred during update of repository %s') \
                     % repo_name, category='error')
                     
         return redirect(url('repo_settings_home', repo_name=changed_name))
@@ -111,7 +113,7 class SettingsController(BaseController)
         # url('repo_settings_delete', repo_name=ID)
         
         repo_model = RepoModel()
-        repo = repo_model.get(repo_name)
+        repo = repo_model.get_by_repo_name(repo_name)
         if not repo:
             h.flash(_('%s repository is not mapped to db perhaps' 
                       ' it was moved or renamed  from the filesystem'
@@ -119,22 +121,22 class SettingsController(BaseController)
                       ' in order to rescan repositories') % repo_name,
                       category='error')
         
-            return redirect(url('hg_home'))
+            return redirect(url('home'))
         try:
             action_logger(self.rhodecode_user, 'user_deleted_repo',
                               repo_name, '', self.sa)            
             repo_model.delete(repo)            
-            invalidate_cache('cached_repo_list')
+            invalidate_cache('get_repo_cached_%s' % repo_name)
             h.flash(_('deleted repository %s') % repo_name, category='success')
         except Exception:
             h.flash(_('An error occurred during deletion of %s') % repo_name,
                     category='error')
         
-        return redirect(url('hg_home'))
+        return redirect(url('home'))
     
     def fork(self, repo_name):
         repo_model = RepoModel()
-        c.repo_info = repo = repo_model.get(repo_name)
+        c.repo_info = repo = repo_model.get_by_repo_name(repo_name)
         if not repo:
             h.flash(_('%s repository is not mapped to db perhaps' 
                       ' it was created or renamed from the filesystem'
@@ -142,7 +144,7 class SettingsController(BaseController)
                       ' in order to rescan repositories') % repo_name,
                       category='error')
         
-            return redirect(url('hg_home'))
+            return redirect(url('home'))
         
         return render('settings/repo_fork.html')
     
@@ -150,17 +152,18 class SettingsController(BaseController)
     
     def fork_create(self, repo_name):
         repo_model = RepoModel()
-        c.repo_info = repo_model.get(repo_name)
-        _form = RepoForkForm()()
+        c.repo_info = repo_model.get_by_repo_name(repo_name)
+        _form = RepoForkForm(old_data={'repo_type':c.repo_info.repo_type})()
         form_result = {}
         try:
             form_result = _form.to_python(dict(request.POST))
             form_result.update({'repo_name':repo_name})
             repo_model.create_fork(form_result, c.rhodecode_user)
-            h.flash(_('fork %s repository as %s task added') \
+            h.flash(_('forked %s repository as %s') \
                       % (repo_name, form_result['fork_name']),
                     category='success')
-            action_logger(self.rhodecode_user, 'user_forked_repo',
+            action_logger(self.rhodecode_user,
+                          'user_forked_repo:%s' % form_result['fork_name'],
                             repo_name, '', self.sa)                                                 
         except formencode.Invalid, errors:
             c.new_repo = errors.value['fork_name']
@@ -172,4 +175,4 class SettingsController(BaseController)
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8")   
-        return redirect(url('hg_home'))
+        return redirect(url('home'))

rhodecode/controllers/shortlog.py

@@ -25,7 +25,7 shortlog controller for pylons
 from pylons import tmpl_context as c, request
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseController, render
-from rhodecode.model.hg_model import HgModel
+from rhodecode.model.scm import ScmModel
 from webhelpers.paginate import Page
 import logging
 log = logging.getLogger(__name__)
@@ -40,7 +40,7 class ShortlogController(BaseController)
         
     def index(self):
         p = int(request.params.get('page', 1))
-        repo = HgModel().get_repo(c.repo_name)
+        repo = ScmModel().get_repo(c.repo_name)
         c.repo_changesets = Page(repo, page=p, items_per_page=20)
         c.shortlog_data = render('shortlog/shortlog_data.html')
         if request.params.get('partial'):

rhodecode/controllers/summary.py

@@ -1,8 +1,15
-#!/usr/bin/env python
-# encoding: utf-8
-# summary controller for pylons
-# Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
-# 
+# -*- coding: utf-8 -*-
+"""
+    package.rhodecode.controllers.summary
+    ~~~~~~~~~~~~~~
+
+    Summary controller for Rhodecode
+    
+    :created_on: Apr 18, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
@@ -17,24 +24,29
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
-"""
-Created on April 18, 2010
-summary controller for pylons
-@author: marcink
-"""
+
+import calendar
+import logging
+from time import mktime
+from datetime import datetime, timedelta
+
+from vcs.exceptions import ChangesetError
+
 from pylons import tmpl_context as c, request, url
+from pylons.i18n.translation import _
+
+from rhodecode.model.scm import ScmModel
+from rhodecode.model.db import Statistics
+
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseController, render
-from rhodecode.lib.utils import OrderedDict
-from rhodecode.model.hg_model import HgModel
-from rhodecode.model.db import Statistics
-from webhelpers.paginate import Page
+from rhodecode.lib.utils import OrderedDict, EmptyChangeset
+
 from rhodecode.lib.celerylib import run_task
 from rhodecode.lib.celerylib.tasks import get_commits_stats
-from datetime import datetime, timedelta
-from time import mktime
-import calendar
-import logging
+
+from webhelpers.paginate import Page
+
 try:
     import json
 except ImportError:
@@ -51,25 +63,44 class SummaryController(BaseController):
         super(SummaryController, self).__before__()
                 
     def index(self):
-        hg_model = HgModel()
-        c.repo_info = hg_model.get_repo(c.repo_name)
-        c.repo_changesets = Page(list(c.repo_info[:10]), page=1, items_per_page=20)
+        scm_model = ScmModel()
+        c.repo_info = scm_model.get_repo(c.repo_name)
+        c.following = scm_model.is_following_repo(c.repo_name,
+                                             c.rhodecode_user.user_id)
+        def url_generator(**kw):
+            return url('shortlog_home', repo_name=c.repo_name, **kw)
+
+        c.repo_changesets = Page(c.repo_info, page=1, items_per_page=10,
+                                 url=url_generator)
+
         e = request.environ
             
-        uri = u'%(protocol)s://%(user)s@%(host)s%(prefix)s/%(repo_name)s' % {
+        if self.rhodecode_user.username == 'default':
+            password = ':default'
+        else:
+            password = ''
+
+        uri = u'%(protocol)s://%(user)s%(password)s@%(host)s%(prefix)s/%(repo_name)s' % {
                                         'protocol': e.get('wsgi.url_scheme'),
                                         'user':str(c.rhodecode_user.username),
+                                        'password':password,
                                         'host':e.get('HTTP_HOST'),
                                         'prefix':e.get('SCRIPT_NAME'),
                                         'repo_name':c.repo_name, }
         c.clone_repo_url = uri
         c.repo_tags = OrderedDict()
         for name, hash in c.repo_info.tags.items()[:10]:
+            try:
             c.repo_tags[name] = c.repo_info.get_changeset(hash)
+            except ChangesetError:
+                c.repo_tags[name] = EmptyChangeset(hash)
         
         c.repo_branches = OrderedDict()
         for name, hash in c.repo_info.branches.items()[:10]:
+            try:
             c.repo_branches[name] = c.repo_info.get_changeset(hash)
+            except ChangesetError:
+                c.repo_branches[name] = EmptyChangeset(hash)
         
         td = datetime.today() + timedelta(days=1) 
         y, m, d = td.year, td.month, td.day
@@ -80,8 +111,11 class SummaryController(BaseController):
                             d, 0, 0, 0, 0, 0, 0,))
         
         ts_max_y = mktime((y, m, d, 0, 0, 0, 0, 0, 0,))
-            
+        if c.repo_info.dbrepo.enable_statistics:
+            c.no_data_msg = _('No data loaded yet')
         run_task(get_commits_stats, c.repo_info.name, ts_min_y, ts_max_y)
+        else:
+            c.no_data_msg = _('Statistics update are disabled for this repository')
         c.ts_min = ts_min_m
         c.ts_max = ts_max_y
         
@@ -91,18 +125,20 class SummaryController(BaseController):
         
         
         if stats and stats.languages:
+            c.no_data = False is c.repo_info.dbrepo.enable_statistics
             lang_stats = json.loads(stats.languages)
             c.commit_data = stats.commit_activity
             c.overview_data = stats.commit_activity_combined
             c.trending_languages = json.dumps(OrderedDict(
                                        sorted(lang_stats.items(), reverse=True,
-                                            key=lambda k: k[1])[:2]
+                                            key=lambda k: k[1])[:10]
                                         )
                                     )
         else:
             c.commit_data = json.dumps({})
-            c.overview_data = json.dumps([[ts_min_y, 0], [ts_max_y, 0] ])
+            c.overview_data = json.dumps([[ts_min_y, 0], [ts_max_y, 10] ])
             c.trending_languages = json.dumps({})
+            c.no_data = True
         
         return render('summary/summary.html')
 

rhodecode/controllers/tags.py

@@ -26,7 +26,7 from pylons import tmpl_context as c
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseController, render
 from rhodecode.lib.utils import OrderedDict
-from rhodecode.model.hg_model import HgModel
+from rhodecode.model.scm import ScmModel
 import logging
 log = logging.getLogger(__name__)
 
@@ -38,7 +38,7 class TagsController(BaseController):
         super(TagsController, self).__before__()
         
     def index(self):
-        hg_model = HgModel()
+        hg_model = ScmModel()
         c.repo_info = hg_model.get_repo(c.repo_name)
         c.repo_tags = OrderedDict()
         for name, hash_ in c.repo_info.tags.items():

rhodecode/lib/app_globals.py

@@ -28,4 +28,4 class Globals(object):
     @LazyProperty
     def base_path(self):
         if self.baseui:
-            return self.paths[0][1].replace('*', '')            
+            return self.paths[0][1]

rhodecode/lib/auth.py

@@ -22,20 +22,21 Created on April 4, 2010
 
 @author: marcink
 """
-from beaker.cache import cache_region
 from pylons import config, session, url, request
 from pylons.controllers.util import abort, redirect
+from rhodecode.lib.exceptions import *
 from rhodecode.lib.utils import get_repo_slug
+from rhodecode.lib.auth_ldap import AuthLdap
 from rhodecode.model import meta
+from rhodecode.model.user import UserModel
 from rhodecode.model.caching_query import FromCache
 from rhodecode.model.db import User, RepoToPerm, Repository, Permission, \
     UserToPerm
-from sqlalchemy.exc import OperationalError
-from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
 import bcrypt
 from decorator import decorator
 import logging
 import random
+import traceback
 
 log = logging.getLogger(__name__) 
 
@@ -74,30 +75,83 def get_crypt_password(password):
 def check_password(password, hashed):
     return bcrypt.hashpw(password, hashed) == hashed
 
-@cache_region('super_short_term', 'cached_user')
-def get_user_cached(username):
-    sa = meta.Session
-    try:
-        user = sa.query(User).filter(User.username == username).one()
-    finally:
-        meta.Session.remove()
-    return user
+def authfunc(environ, username, password):
+    """
+    Dummy authentication function used in Mercurial/Git/ and access control,
+    
+    :param environ: needed only for using in Basic auth
+    """
+    return authenticate(username, password)
+
+
+def authenticate(username, password):
+    """
+    Authentication function used for access control,
+    firstly checks for db authentication then if ldap is enabled for ldap
+    authentication, also creates ldap user if not in database
 
-def authfunc(environ, username, password):
-    try:
-        user = get_user_cached(username)
-    except (NoResultFound, MultipleResultsFound, OperationalError), e:
-        log.error(e)
-        user = None
+    :param username: username
+    :param password: password
+    """
+    user_model = UserModel()
+    user = user_model.get_by_username(username, cache=False)
         
-    if user:
+    log.debug('Authenticating user using RhodeCode account')
+    if user is not None and user.is_ldap is False:
         if user.active:
-            if user.username == username and check_password(password, user.password):
+
+            if user.username == 'default' and user.active:
+                log.info('user %s authenticated correctly as anonymous user',
+                         username)
+                return True
+
+            elif user.username == username and check_password(password, user.password):
                 log.info('user %s authenticated correctly', username)
                 return True
         else:
-            log.error('user %s is disabled', username)
+            log.warning('user %s is disabled', username)
+
+    else:
+        log.debug('Regular authentication failed')
+        user_obj = user_model.get_by_username(username, cache=False,
+                                            case_insensitive=True)
+
+        if user_obj is not None and user_obj.is_ldap is False:
+            log.debug('this user already exists as non ldap')
+            return False
+
+        from rhodecode.model.settings import SettingsModel
+        ldap_settings = SettingsModel().get_ldap_settings()
             
+        #======================================================================
+        # FALLBACK TO LDAP AUTH IN ENABLE                
+        #======================================================================
+        if ldap_settings.get('ldap_active', False):
+            log.debug("Authenticating user using ldap")
+            kwargs = {
+                  'server':ldap_settings.get('ldap_host', ''),
+                  'base_dn':ldap_settings.get('ldap_base_dn', ''),
+                  'port':ldap_settings.get('ldap_port'),
+                  'bind_dn':ldap_settings.get('ldap_dn_user'),
+                  'bind_pass':ldap_settings.get('ldap_dn_pass'),
+                  'use_ldaps':ldap_settings.get('ldap_ldaps'),
+                  'ldap_version':3,
+                  }
+            log.debug('Checking for ldap authentication')
+            try:
+                aldap = AuthLdap(**kwargs)
+                res = aldap.authenticate_ldap(username, password)
+                log.debug('Got ldap response %s', res)
+
+                if user_model.create_ldap(username, password):
+                    log.info('created new ldap user')
+
+                return True
+            except (LdapUsernameError, LdapPasswordError,):
+                pass
+            except (Exception,):
+                log.error(traceback.format_exc())
+                pass
     return False
 
 class  AuthUser(object):
@@ -114,6 +168,8 class AuthUser(object):
         self.is_admin = False
         self.permissions = {}
 
+    def __repr__(self):
+        return "<AuthUser('id:%s:%s')>" % (self.user_id, self.username)
 
 def set_available_permissions(config):
     """
@@ -125,8 +181,10 def set_available_permissions(config):
     """
     log.info('getting information about all available permissions')
     try:
-        sa = meta.Session
+        sa = meta.Session()
         all_perms = sa.query(Permission).all()
+    except:
+        pass
     finally:
         meta.Session.remove()
     
@@ -135,26 +193,6 def set_available_permissions(config):
 def set_base_path(config):
     config['base_path'] = config['pylons.app_globals'].base_path
 
-def fill_data(user):
-    """
-    Fills user data with those from database and log out user if not present
-    in database
-    :param user:
-    """
-    sa = meta.Session
-    dbuser = sa.query(User).options(FromCache('sql_cache_short',
-                                              'getuser_%s' % user.user_id))\
-        .get(user.user_id)
-    if dbuser:
-        user.username = dbuser.username
-        user.is_admin = dbuser.admin
-        user.name = dbuser.name
-        user.lastname = dbuser.lastname
-        user.email = dbuser.email
-    else:
-        user.is_authenticated = False
-    meta.Session.remove()
-    return user
             
 def fill_perms(user):
     """
@@ -162,16 +200,14 def fill_perms(user):
     :param user:
     """
     
-    sa = meta.Session
+    sa = meta.Session()
     user.permissions['repositories'] = {}
     user.permissions['global'] = set()
     
     #===========================================================================
     # fetch default permissions
     #===========================================================================
-    default_user = sa.query(User)\
-        .options(FromCache('sql_cache_short','getuser_%s' % 'default'))\
-        .filter(User.username == 'default').scalar()
+    default_user = UserModel().get_by_username('default', cache=True)
                                             
     default_perms = sa.query(RepoToPerm, Repository, Permission)\
         .join((Repository, RepoToPerm.repository_id == Repository.repo_id))\
@@ -195,8 +231,8 def fill_perms(user):
         
         #default global
         default_global_perms = sa.query(UserToPerm)\
-            .filter(UserToPerm.user == sa.query(User).filter(User.username == 
-            'default').one())
+            .filter(UserToPerm.user == sa.query(User)\
+                   .filter(User.username == 'default').one())
         
         for perm in default_global_perms:
             user.permissions['global'].add(perm.permission.permission_name)
@@ -237,8 +273,21 def get_user(session):
     :param session:
     """
     user = session.get('rhodecode_user', AuthUser())
+    #if the user is not logged in we check for anonymous access
+    #if user is logged and it's a default user check if we still have anonymous
+    #access enabled
+    if user.user_id is None or user.username == 'default':
+        anonymous_user = UserModel().get_by_username('default', cache=True)
+        if anonymous_user.active is True:
+            #then we set this user is logged in
+            user.is_authenticated = True
+            user.user_id = anonymous_user.user_id
+        else:
+            user.is_authenticated = False
+
     if user.is_authenticated:
-        user = fill_data(user)
+        user = UserModel().fill_data(user)
+
     user = fill_perms(user)
     session['rhodecode_user'] = user
     session.save()
@@ -248,7 +297,8 def get_user(session):
 # CHECK DECORATORS
 #===============================================================================
 class LoginRequired(object):
-    """Must be logged in to execute this function else redirect to login page"""
+    """Must be logged in to execute this function else 
+    redirect to login page"""
    
     def __call__(self, func):
         return decorator(self.__wrapper, func)
@@ -273,6 +323,31 class LoginRequired(object):
             log.debug('redirecting to login page with %s', p)                
             return redirect(url('login_home', came_from=p))
 
+class NotAnonymous(object):
+    """Must be logged in to execute this function else 
+    redirect to login page"""
+
+    def __call__(self, func):
+        return decorator(self.__wrapper, func)
+
+    def __wrapper(self, func, *fargs, **fkwargs):
+        user = session.get('rhodecode_user', AuthUser())
+        log.debug('Checking if user is not anonymous')
+
+        anonymous = user.username == 'default'
+
+        if anonymous:
+            p = ''
+            if request.environ.get('SCRIPT_NAME') != '/':
+                p += request.environ.get('SCRIPT_NAME')
+
+            p += request.environ.get('PATH_INFO')
+            if request.environ.get('QUERY_STRING'):
+                p += '?' + request.environ.get('QUERY_STRING')
+            return redirect(url('login_home', came_from=p))
+        else:
+            return func(*fargs, **fkwargs)
+
 class PermsDecorator(object):
     """Base class for decorators"""
     
@@ -292,18 +367,19 class PermsDecorator(object):
 #        _wrapper.__name__ = func.__name__
 #        _wrapper.__dict__.update(func.__dict__)
 #        _wrapper.__doc__ = func.__doc__
-
-        self.user_perms = session.get('rhodecode_user', AuthUser()).permissions
-        log.debug('checking %s permissions %s for %s',
-           self.__class__.__name__, self.required_perms, func.__name__)
+        self.user = session.get('rhodecode_user', AuthUser())
+        self.user_perms = self.user.permissions
+        log.debug('checking %s permissions %s for %s %s',
+           self.__class__.__name__, self.required_perms, func.__name__,
+               self.user)
         
         if self.check_permissions():
-            log.debug('Permission granted for %s', func.__name__)
+            log.debug('Permission granted for %s %s', func.__name__, self.user)
             
             return func(*fargs, **fkwargs)
         
         else:
-            log.warning('Permission denied for %s', func.__name__)
+            log.warning('Permission denied for %s %s', func.__name__, self.user)
             #redirect with forbidden ret code
             return abort(403)
 
@@ -389,16 +465,17 class PermsFunction(object):
             return False
         self.user_perms = user.permissions
         self.granted_for = user.username        
-        log.debug('checking %s %s', self.__class__.__name__, self.required_perms)            
+        log.debug('checking %s %s %s', self.__class__.__name__,
+                  self.required_perms, user)
         
         if self.check_permissions():
-            log.debug('Permission granted for %s @%s', self.granted_for,
-                      check_Location)
+            log.debug('Permission granted for %s @ %s %s', self.granted_for,
+                      check_Location, user)
             return True
         
         else:
-            log.warning('Permission denied for %s @%s', self.granted_for,
-                        check_Location)
+            log.warning('Permission denied for %s @ %s %s', self.granted_for,
+                        check_Location, user)
             return False 
     
     def check_permissions(self):

rhodecode/lib/base.py

@@ -9,8 +9,8 from rhodecode import __version__
 from rhodecode.lib import auth
 from rhodecode.lib.utils import get_repo_slug
 from rhodecode.model import meta
-from rhodecode.model.hg_model import _get_repos_cached, \
-    _get_repos_switcher_cached
+from rhodecode.model.scm import ScmModel
+from rhodecode import BACKENDS
 
 class BaseController(WSGIController):
     
@@ -18,20 +18,26 class BaseController(WSGIController):
         c.rhodecode_version = __version__
         c.rhodecode_name = config['rhodecode_title']
         c.repo_name = get_repo_slug(request)
-        c.cached_repo_list = _get_repos_cached()
-        c.repo_switcher_list = _get_repos_switcher_cached(c.cached_repo_list)
+        c.cached_repo_list = ScmModel().get_repos()
+        c.backends = BACKENDS.keys()
+        self.cut_off_limit = int(config['cut_off_limit'])
+        self.sa = meta.Session()
+        scm_model = ScmModel(self.sa)
+        #c.unread_journal = scm_model.get_unread_journal()
         
         if c.repo_name:
-            cached_repo = c.cached_repo_list.get(c.repo_name)
-            
+            cached_repo = scm_model.get(c.repo_name)
             if cached_repo:
                 c.repository_tags = cached_repo.tags
                 c.repository_branches = cached_repo.branches
+                c.repository_followers = scm_model.get_followers(cached_repo.dbrepo.repo_id)
+                c.repository_forks = scm_model.get_forks(cached_repo.dbrepo.repo_id)
             else:
                 c.repository_tags = {}
                 c.repository_branches = {}
+                c.repository_followers = 0
+                c.repository_forks = 0
                     
-        self.sa = meta.Session
     
     def __call__(self, environ, start_response):
         """Invoke the Controller"""

rhodecode/lib/celerylib/__init__.py

@@ -1,14 +1,54
-from rhodecode.lib.pidlock import DaemonLock, LockHeld
-from vcs.utils.lazy import LazyProperty
-from decorator import decorator
-import logging
+# -*- coding: utf-8 -*-
+"""
+    package.rhodecode.lib.celerylib.__init__
+    ~~~~~~~~~~~~~~
+
+    celery libs for RhodeCode
+    
+    :created_on: Nov 27, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your opinion) any later version of the license.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+
 import os
 import sys
+import socket
 import traceback
+import logging
+
 from hashlib import md5
-import socket
+from decorator import decorator
+from vcs.utils.lazy import LazyProperty
+
+from rhodecode.lib.pidlock import DaemonLock, LockHeld
+
+from pylons import  config
+
 log = logging.getLogger(__name__)
 
+def str2bool(v):
+    return v.lower() in ["yes", "true", "t", "1"] if v else None
+
+try:
+    CELERY_ON = str2bool(config['app_conf'].get('use_celery'))
+except KeyError:
+    CELERY_ON = False
+
 class ResultWrapper(object):
     def __init__(self, task):
         self.task = task
@@ -18,27 +58,22 class ResultWrapper(object):
         return self.task
 
 def run_task(task, *args, **kwargs):
+    if CELERY_ON:
     try:
         t = task.delay(*args, **kwargs)
-        log.info('running task %s', t.task_id)
+            log.info('running task %s:%s', t.task_id, task)
         return t
     except socket.error, e:
-
-        try:
-            conn_failed = e.errno == 111
-        except AttributeError:
-            conn_failed = False
-
-        if  conn_failed:
+            if  e.errno == 111:
             log.debug('Unable to connect to celeryd. Sync execution')
         else:
-            log.debug('Unable to connect to celeryd. Sync execution')
-
+                log.error(traceback.format_exc())
     except KeyError, e:
             log.debug('Unable to connect to celeryd. Sync execution')
     except Exception, e:
         log.error(traceback.format_exc())
 
+    log.debug('executing task %s in sync mode', task)
     return ResultWrapper(task(*args, **kwargs))
 
 

rhodecode/lib/celerylib/tasks.py

@@ -1,14 +1,26
 from celery.decorators import task
 
+import os
+import traceback
+from time import mktime
 from operator import itemgetter
+
+from pylons import config
 from pylons.i18n.translation import _
-from rhodecode.lib.celerylib import run_task, locked_task
+
+from rhodecode.lib.celerylib import run_task, locked_task, str2bool
 from rhodecode.lib.helpers import person
 from rhodecode.lib.smtp_mailer import SmtpMailer
-from rhodecode.lib.utils import OrderedDict
-from time import mktime
-from vcs.backends.hg import MercurialRepository
-import traceback
+from rhodecode.lib.utils import OrderedDict, add_cache
+from rhodecode.model import init_model
+from rhodecode.model import meta
+from rhodecode.model.db import RhodeCodeUi
+
+from vcs.backends import get_repo
+
+from sqlalchemy import engine_from_config
+
+add_cache(config)
 
 try:
     import json
@@ -16,90 +28,50 except ImportError:
     #python 2.5 compatibility
     import simplejson as json
 
-try:
-    from celeryconfig import PYLONS_CONFIG as config
-    celery_on = True
-except ImportError:
-    #if celeryconfig is not present let's just load our pylons
-    #config instead
-    from pylons import config
-    celery_on = False
-
-
 __all__ = ['whoosh_index', 'get_commits_stats',
            'reset_user_password', 'send_email']
 
+CELERY_ON = str2bool(config['app_conf'].get('use_celery'))
+
 def get_session():
-    if celery_on:
-        from sqlalchemy import engine_from_config
-        from sqlalchemy.orm import sessionmaker, scoped_session
-        engine = engine_from_config(dict(config.items('app:main')), 'sqlalchemy.db1.')
-        sa = scoped_session(sessionmaker(bind=engine))
-    else:
-        #If we don't use celery reuse our current application Session
-        from rhodecode.model.meta import Session
-        sa = Session
-        
+    if CELERY_ON:
+        engine = engine_from_config(config, 'sqlalchemy.db1.')
+        init_model(engine)
+    sa = meta.Session()
     return sa
 
-def get_hg_settings():
-    from rhodecode.model.db import RhodeCodeSettings
-    sa = get_session()
-    ret = sa.query(RhodeCodeSettings).all()
-        
-    if not ret:
-        raise Exception('Could not get application settings !')
-    settings = {}
-    for each in ret:
-        settings['rhodecode_' + each.app_settings_name] = each.app_settings_value    
-    
-    return settings
-
-def get_hg_ui_settings():
-    from rhodecode.model.db import RhodeCodeUi
+def get_repos_path():
     sa = get_session()
-    ret = sa.query(RhodeCodeUi).all()
-        
-    if not ret:
-        raise Exception('Could not get application ui settings !')
-    settings = {}
-    for each in ret:
-        k = each.ui_key
-        v = each.ui_value
-        if k == '/':
-            k = 'root_path'
-        
-        if k.find('.') != -1:
-            k = k.replace('.', '_')
-        
-        if each.ui_section == 'hooks':
-            v = each.ui_active
-        
-        settings[each.ui_section + '_' + k] = v  
-    
-    return settings   
+    q = sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
+    return q.ui_value
 
 @task
 @locked_task
 def whoosh_index(repo_location, full_index):
     log = whoosh_index.get_logger()
     from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
-    WhooshIndexingDaemon(repo_location=repo_location).run(full_index=full_index)
+    index_location = config['index_dir']
+    WhooshIndexingDaemon(index_location=index_location,
+                         repo_location=repo_location, sa=get_session())\
+                         .run(full_index=full_index)
 
 @task
 @locked_task
 def get_commits_stats(repo_name, ts_min_y, ts_max_y):
     from rhodecode.model.db import Statistics, Repository
     log = get_commits_stats.get_logger()
-    author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty
+
+    #for js data compatibilty
+    author_key_cleaner = lambda k: person(k).replace('"', "")
     
     commits_by_day_author_aggregate = {}
     commits_by_day_aggregate = {}
-    repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '')
-    repo = MercurialRepository(repos_path + repo_name)
+    repos_path = get_repos_path()
+    p = os.path.join(repos_path, repo_name)
+    repo = get_repo(p)
 
     skip_date_limit = True
-    parse_limit = 350 #limit for single task changeset parsing optimal for
+    parse_limit = 250 #limit for single task changeset parsing optimal for
     last_rev = 0
     last_cs = None
     timegetter = itemgetter('time')
@@ -116,8 +88,8 def get_commits_stats(repo_name, ts_min_
         return True
     
     if last_rev == repo.revisions[-1] and len(repo.revisions) > 1:
-        #pass silently without any work if we're not on first revision or current
-        #state of parsing revision(from db marker) is the last revision
+        #pass silently without any work if we're not on first revision or 
+        #current state of parsing revision(from db marker) is the last revision
         return True
     
     if cur_stats:
@@ -127,13 +99,14 def get_commits_stats(repo_name, ts_min_
         commits_by_day_author_aggregate = json.loads(cur_stats.commit_activity)
     
     log.debug('starting parsing %s', parse_limit)
-    for cnt, rev in enumerate(repo.revisions[last_rev:]):
+    lmktime = mktime
+
+    last_rev = last_rev + 1 if last_rev > 0 else last_rev
+    for rev in repo.revisions[last_rev:last_rev + parse_limit]:
         last_cs = cs = repo.get_changeset(rev)
-        k = '%s-%s-%s' % (cs.date.timetuple()[0], cs.date.timetuple()[1],
-                          cs.date.timetuple()[2])
-        timetupple = [int(x) for x in k.split('-')]
-        timetupple.extend([0 for _ in xrange(6)])
-        k = mktime(timetupple)
+        k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1],
+                      cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0])
+
         if commits_by_day_author_aggregate.has_key(author_key_cleaner(cs.author)):
             try:
                 l = [timegetter(x) for x in commits_by_day_author_aggregate\
@@ -183,15 +156,7 def get_commits_stats(repo_name, ts_min_
         else:
             commits_by_day_aggregate[k] = 1
         
-        if cnt >= parse_limit:
-            #don't fetch to much data since we can freeze application
-            break
-
-    overview_data = []
-    for k, v in commits_by_day_aggregate.items():
-        overview_data.append([k, v])
-    overview_data = sorted(overview_data, key=itemgetter(0))
-        
+    overview_data = sorted(commits_by_day_aggregate.items(), key=itemgetter(0))
     if not commits_by_day_author_aggregate:
         commits_by_day_author_aggregate[author_key_cleaner(repo.contact)] = {
             "label":author_key_cleaner(repo.contact),
@@ -208,6 +173,7 def get_commits_stats(repo_name, ts_min_
     log.debug('revisions to parse %s', leftovers)
     
     if last_rev == 0 or leftovers < parse_limit:    
+        log.debug('getting code trending stats')
         stats.languages = json.dumps(__get_codes_stats(repo_name))
         
     stats.repository = dbrepo
@@ -258,19 +224,33 def reset_user_password(user_email):
     except:
         log.error('Failed to update user password')
         log.error(traceback.format_exc())
+
     return True
 
 @task    
 def send_email(recipients, subject, body):
+    """
+    Sends an email with defined parameters from the .ini files.
+    
+    
+    :param recipients: list of recipients, it this is empty the defined email
+        address from field 'email_to' is used instead
+    :param subject: subject of the mail
+    :param body: body of the mail
+    """
     log = send_email.get_logger()
-    email_config = dict(config.items('DEFAULT')) 
+    email_config = config
+
+    if not recipients:
+        recipients = [email_config.get('email_to')]
+
     mail_from = email_config.get('app_email_from')
     user = email_config.get('smtp_username')
     passwd = email_config.get('smtp_password')
     mail_server = email_config.get('smtp_server')
     mail_port = email_config.get('smtp_port')
-    tls = email_config.get('smtp_use_tls')
-    ssl = False
+    tls = str2bool(email_config.get('smtp_use_tls'))
+    ssl = str2bool(email_config.get('smtp_use_ssl'))
     
     try:
         m = SmtpMailer(mail_from, user, passwd, mail_server,
@@ -284,45 +264,96 def send_email(recipients, subject, body
 
 @task
 def create_repo_fork(form_data, cur_user):
-    import os
-    from rhodecode.model.repo_model import RepoModel
-    sa = get_session()
-    rm = RepoModel(sa)
-    
-    rm.create(form_data, cur_user, just_db=True, fork=True)
+    from rhodecode.model.repo import RepoModel
+    from vcs import get_backend
+    log = create_repo_fork.get_logger()
+    repo_model = RepoModel(get_session())
+    repo_model.create(form_data, cur_user, just_db=True, fork=True)
+    repo_name = form_data['repo_name']
+    repos_path = get_repos_path()
+    repo_path = os.path.join(repos_path, repo_name)
+    repo_fork_path = os.path.join(repos_path, form_data['fork_name'])
+    alias = form_data['repo_type']
     
-    repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '')
-    repo_path = os.path.join(repos_path, form_data['repo_name'])
-    repo_fork_path = os.path.join(repos_path, form_data['fork_name'])
-    
-    MercurialRepository(str(repo_fork_path), True, clone_url=str(repo_path))
-
+    log.info('creating repo fork %s as %s', repo_name, repo_path)
+    backend = get_backend(alias)
+    backend(str(repo_fork_path), create=True, src_url=str(repo_path))
     
 def __get_codes_stats(repo_name):
-    LANGUAGES_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c',
-                    'cfg', 'cfm', 'cpp', 'cs', 'diff', 'do', 'el', 'erl',
-                    'h', 'java', 'js', 'jsp', 'jspx', 'lisp',
-                    'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3',
-                    'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh',
-                    'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt',
-                    'yaws']
-    repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '')
-    repo = MercurialRepository(repos_path + repo_name)
+    LANGUAGES_EXTENSIONS_MAP = {'scm': 'Scheme', 'asmx': 'VbNetAspx', 'Rout':
+    'RConsole', 'rest': 'Rst', 'abap': 'ABAP', 'go': 'Go', 'phtml': 'HtmlPhp',
+    'ns2': 'Newspeak', 'xml': 'EvoqueXml', 'sh-session': 'BashSession', 'ads':
+    'Ada', 'clj': 'Clojure', 'll': 'Llvm', 'ebuild': 'Bash', 'adb': 'Ada',
+    'ada': 'Ada', 'c++-objdump': 'CppObjdump', 'aspx':
+    'VbNetAspx', 'ksh': 'Bash', 'coffee': 'CoffeeScript', 'vert': 'GLShader',
+    'Makefile.*': 'Makefile', 'di': 'D', 'dpatch': 'DarcsPatch', 'rake':
+    'Ruby', 'moo': 'MOOCode', 'erl-sh': 'ErlangShell', 'geo': 'GLShader',
+    'pov': 'Povray', 'bas': 'VbNet', 'bat': 'Batch', 'd': 'D', 'lisp':
+    'CommonLisp', 'h': 'C', 'rbx': 'Ruby', 'tcl': 'Tcl', 'c++': 'Cpp', 'md':
+    'MiniD', '.vimrc': 'Vim', 'xsd': 'Xml', 'ml': 'Ocaml', 'el': 'CommonLisp',
+    'befunge': 'Befunge', 'xsl': 'Xslt', 'pyx': 'Cython', 'cfm':
+    'ColdfusionHtml', 'evoque': 'Evoque', 'cfg': 'Ini', 'htm': 'Html',
+    'Makefile': 'Makefile', 'cfc': 'ColdfusionHtml', 'tex': 'Tex', 'cs':
+    'CSharp', 'mxml': 'Mxml', 'patch': 'Diff', 'apache.conf': 'ApacheConf',
+    'scala': 'Scala', 'applescript': 'AppleScript', 'GNUmakefile': 'Makefile',
+    'c-objdump': 'CObjdump', 'lua': 'Lua', 'apache2.conf': 'ApacheConf', 'rb':
+    'Ruby', 'gemspec': 'Ruby', 'rl': 'RagelObjectiveC', 'vala': 'Vala', 'tmpl':
+    'Cheetah', 'bf': 'Brainfuck', 'plt': 'Gnuplot', 'G': 'AntlrRuby', 'xslt':
+    'Xslt', 'flxh': 'Felix', 'asax': 'VbNetAspx', 'Rakefile': 'Ruby', 'S': 'S',
+    'wsdl': 'Xml', 'js': 'Javascript', 'autodelegate': 'Myghty', 'properties':
+    'Ini', 'bash': 'Bash', 'c': 'C', 'g': 'AntlrRuby', 'r3': 'Rebol', 's':
+    'Gas', 'ashx': 'VbNetAspx', 'cxx': 'Cpp', 'boo': 'Boo', 'prolog': 'Prolog',
+    'sqlite3-console': 'SqliteConsole', 'cl': 'CommonLisp', 'cc': 'Cpp', 'pot':
+    'Gettext', 'vim': 'Vim', 'pxi': 'Cython', 'yaml': 'Yaml', 'SConstruct':
+    'Python', 'diff': 'Diff', 'txt': 'Text', 'cw': 'Redcode', 'pxd': 'Cython',
+    'plot': 'Gnuplot', 'java': 'Java', 'hrl': 'Erlang', 'py': 'Python',
+    'makefile': 'Makefile', 'squid.conf': 'SquidConf', 'asm': 'Nasm', 'toc':
+    'Tex', 'kid': 'Genshi', 'rhtml': 'Rhtml', 'po': 'Gettext', 'pl': 'Prolog',
+    'pm': 'Perl', 'hx': 'Haxe', 'ascx': 'VbNetAspx', 'ooc': 'Ooc', 'asy':
+    'Asymptote', 'hs': 'Haskell', 'SConscript': 'Python', 'pytb':
+    'PythonTraceback', 'myt': 'Myghty', 'hh': 'Cpp', 'R': 'S', 'aux': 'Tex',
+    'rst': 'Rst', 'cpp-objdump': 'CppObjdump', 'lgt': 'Logtalk', 'rss': 'Xml',
+    'flx': 'Felix', 'b': 'Brainfuck', 'f': 'Fortran', 'rbw': 'Ruby',
+    '.htaccess': 'ApacheConf', 'cxx-objdump': 'CppObjdump', 'j': 'ObjectiveJ',
+    'mll': 'Ocaml', 'yml': 'Yaml', 'mu': 'MuPAD', 'r': 'Rebol', 'ASM': 'Nasm',
+    'erl': 'Erlang', 'mly': 'Ocaml', 'mo': 'Modelica', 'def': 'Modula2', 'ini':
+    'Ini', 'control': 'DebianControl', 'vb': 'VbNet', 'vapi': 'Vala', 'pro':
+    'Prolog', 'spt': 'Cheetah', 'mli': 'Ocaml', 'as': 'ActionScript3', 'cmd':
+    'Batch', 'cpp': 'Cpp', 'io': 'Io', 'tac': 'Python', 'haml': 'Haml', 'rkt':
+    'Racket', 'st':'Smalltalk', 'inc': 'Povray', 'pas': 'Delphi', 'cmake':
+    'CMake', 'csh':'Tcsh', 'hpp': 'Cpp', 'feature': 'Gherkin', 'html': 'Html',
+    'php':'Php', 'php3':'Php', 'php4':'Php', 'php5':'Php', 'xhtml': 'Html',
+    'hxx': 'Cpp', 'eclass': 'Bash', 'css': 'Css',
+    'frag': 'GLShader', 'd-objdump': 'DObjdump', 'weechatlog': 'IrcLogs',
+    'tcsh': 'Tcsh', 'objdump': 'Objdump', 'pyw': 'Python', 'h++': 'Cpp',
+    'py3tb': 'Python3Traceback', 'jsp': 'Jsp', 'sql': 'Sql', 'mak': 'Makefile',
+    'php': 'Php', 'mao': 'Mako', 'man': 'Groff', 'dylan': 'Dylan', 'sass':
+    'Sass', 'cfml': 'ColdfusionHtml', 'darcspatch': 'DarcsPatch', 'tpl':
+    'Smarty', 'm': 'ObjectiveC', 'f90': 'Fortran', 'mod': 'Modula2', 'sh':
+    'Bash', 'lhs': 'LiterateHaskell', 'sources.list': 'SourcesList', 'axd':
+    'VbNetAspx', 'sc': 'Python'}
+
+    repos_path = get_repos_path()
+    p = os.path.join(repos_path, repo_name)
+    repo = get_repo(p)
     tip = repo.get_changeset()
+    code_stats = {}
     
-    code_stats = {}
-    for topnode, dirs, files in tip.walk('/'):
-        for f in files:
-            k = f.mimetype
-            if f.extension in LANGUAGES_EXTENSIONS:
-                if code_stats.has_key(k):
-                    code_stats[k] += 1
+    def aggregate(cs):
+        for f in cs[2]:
+            ext = f.extension
+            key = LANGUAGES_EXTENSIONS_MAP.get(ext, ext)
+            key = key or ext
+            if ext in LANGUAGES_EXTENSIONS_MAP.keys() and not f.is_binary:
+                if code_stats.has_key(key):
+                    code_stats[key] += 1
                 else:
-                    code_stats[k] = 1
+                    code_stats[key] = 1
+
+    map(aggregate, tip.walk('/'))
                     
     return code_stats or {}
 
 
             
 
-

rhodecode/lib/db_manage.py

@@ -1,8 +1,16
-#!/usr/bin/env python
-# encoding: utf-8
-# database management for RhodeCode
-# Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
-#
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.lib.db_manage
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Database creation, and setup module for RhodeCode. Used for creation
+    of database as well as for migration operations
+    
+    :created_on: Apr 10, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
@@ -18,51 +26,50
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-"""
-Created on April 10, 2010
-database management and creation for RhodeCode
-@author: marcink
-"""
-
-from os.path import dirname as dn, join as jn
 import os
 import sys
 import uuid
+import logging
+from os.path import dirname as dn, join as jn
+
+from rhodecode import __dbversion__
+from rhodecode.model import meta
 
 from rhodecode.lib.auth import get_crypt_password
 from rhodecode.lib.utils import ask_ok
 from rhodecode.model import init_model
 from rhodecode.model.db import User, Permission, RhodeCodeUi, RhodeCodeSettings, \
-    UserToPerm
-from rhodecode.model import meta
+    UserToPerm, DbMigrateVersion
+
 from sqlalchemy.engine import create_engine
-import logging
 
 log = logging.getLogger(__name__)
 
 class DbManage(object):
-    def __init__(self, log_sql, dbname, root, tests=False):
-        self.dbname = dbname
+    def __init__(self, log_sql, dbconf, root, tests=False):
+        self.dbname = dbconf.split('/')[-1]
         self.tests = tests
         self.root = root
-        dburi = 'sqlite:////%s' % jn(self.root, self.dbname)
-        engine = create_engine(dburi, echo=log_sql) 
+        self.dburi = dbconf
+        engine = create_engine(self.dburi, echo=log_sql)
         init_model(engine)
-        self.sa = meta.Session
+        self.sa = meta.Session()
         self.db_exists = False
     
     def check_for_db(self, override):
         db_path = jn(self.root, self.dbname)
+        if self.dburi.startswith('sqlite'):
         log.info('checking for existing db in %s', db_path)
         if os.path.isfile(db_path):
+
             self.db_exists = True
             if not override:
                 raise Exception('database already exists')
 
     def create_tables(self, override=False):
+        """Create a auth database
         """
-        Create a auth database
-        """
+
         self.check_for_db(override)
         if self.db_exists:
             log.info("database exist and it's going to be destroyed")
@@ -78,6 +85,135 class DbManage(object):
         meta.Base.metadata.create_all(checkfirst=checkfirst)
         log.info('Created tables for %s', self.dbname)
     
+
+
+    def set_db_version(self):
+        try:
+            ver = DbMigrateVersion()
+            ver.version = __dbversion__
+            ver.repository_id = 'rhodecode_db_migrations'
+            ver.repository_path = 'versions'
+            self.sa.add(ver)
+            self.sa.commit()
+        except:
+            self.sa.rollback()
+            raise
+        log.info('db version set to: %s', __dbversion__)
+
+
+    def upgrade(self):
+        """Upgrades given database schema to given revision following 
+        all needed steps,  
+        
+        :param revision: revision to upgrade to
+        """
+
+        from rhodecode.lib.dbmigrate.migrate.versioning import api
+        from rhodecode.lib.dbmigrate.migrate.exceptions import \
+            DatabaseNotControlledError
+
+        upgrade = ask_ok('You are about to perform database upgrade, make '
+                         'sure You backed up your database before. '
+                         'Continue ? [y/n]')
+        if not upgrade:
+            sys.exit('Nothing done')
+
+        repository_path = jn(dn(dn(dn(os.path.realpath(__file__)))),
+                             'rhodecode/lib/dbmigrate')
+        db_uri = self.dburi
+
+        try:
+            curr_version = api.db_version(db_uri, repository_path)
+            msg = ('Found current database under version'
+                 ' control with version %s' % curr_version)
+
+        except (RuntimeError, DatabaseNotControlledError), e:
+            curr_version = 1
+            msg = ('Current database is not under version control. Setting'
+                   ' as version %s' % curr_version)
+            api.version_control(db_uri, repository_path, curr_version)
+
+        print (msg)
+
+        if curr_version == __dbversion__:
+            sys.exit('This database is already at the newest version')
+
+        #======================================================================
+        # UPGRADE STEPS
+        #======================================================================
+        class UpgradeSteps(object):
+
+            def __init__(self, klass):
+                self.klass = klass
+
+            def step_0(self):
+                #step 0 is the schema upgrade, and than follow proper upgrades
+                print ('attempting to do database upgrade to version %s' \
+                                % __dbversion__)
+                api.upgrade(db_uri, repository_path, __dbversion__)
+                print ('Schema upgrade completed')
+
+            def step_1(self):
+                pass
+
+            def step_2(self):
+                print ('Patching repo paths for newer version of RhodeCode')
+                self.klass.fix_repo_paths()
+
+                print ('Patching default user of RhodeCode')
+                self.klass.fix_default_user()
+
+                log.info('Changing ui settings')
+                self.klass.create_ui_settings()
+
+
+        upgrade_steps = [0] + range(curr_version + 1, __dbversion__ + 1)
+
+        #CALL THE PROPER ORDER OF STEPS TO PERFORM FULL UPGRADE
+        for step in upgrade_steps:
+            print ('performing upgrade step %s' % step)
+            callable = getattr(UpgradeSteps(self), 'step_%s' % step)()
+
+
+
+    def fix_repo_paths(self):
+        """Fixes a old rhodecode version path into new one without a '*'
+        """
+
+        paths = self.sa.query(RhodeCodeUi)\
+                .filter(RhodeCodeUi.ui_key == '/')\
+                .scalar()
+
+        paths.ui_value = paths.ui_value.replace('*', '')
+
+        try:
+            self.sa.add(paths)
+            self.sa.commit()
+        except:
+            self.sa.rollback()
+            raise
+
+    def fix_default_user(self):
+        """Fixes a old default user with some 'nicer' default values,
+        used mostly for anonymous access
+        """
+        def_user = self.sa.query(User)\
+                .filter(User.username == 'default')\
+                .one()
+
+        def_user.name = 'Anonymous'
+        def_user.lastname = 'User'
+        def_user.email = '[e-mail]'
+
+        try:
+            self.sa.add(def_user)
+            self.sa.commit()
+        except:
+            self.sa.rollback()
+            raise
+
+
+
     def admin_prompt(self, second=False):
         if not self.tests:
             import getpass
@@ -113,7 +249,72 class DbManage(object):
             self.create_user('test_regular', 'test12', '[e-mail]', False)
             self.create_user('test_regular2', 'test12', '[e-mail]', False)
             
+    def create_ui_settings(self):
+        """Creates ui settings, fills out hooks
+        and disables dotencode
         
+        """
+        #HOOKS
+        hooks1_key = 'changegroup.update'
+        hooks1_ = self.sa.query(RhodeCodeUi)\
+            .filter(RhodeCodeUi.ui_key == hooks1_key).scalar()
+
+        hooks1 = RhodeCodeUi() if hooks1_ is None else hooks1_
+        hooks1.ui_section = 'hooks'
+        hooks1.ui_key = hooks1_key
+        hooks1.ui_value = 'hg update >&2'
+        hooks1.ui_active = False
+
+        hooks2_key = 'changegroup.repo_size'
+        hooks2_ = self.sa.query(RhodeCodeUi)\
+            .filter(RhodeCodeUi.ui_key == hooks2_key).scalar()
+
+        hooks2 = RhodeCodeUi() if hooks2_ is None else hooks2_
+        hooks2.ui_section = 'hooks'
+        hooks2.ui_key = hooks2_key
+        hooks2.ui_value = 'python:rhodecode.lib.hooks.repo_size'
+
+        hooks3 = RhodeCodeUi()
+        hooks3.ui_section = 'hooks'
+        hooks3.ui_key = 'pretxnchangegroup.push_logger'
+        hooks3.ui_value = 'python:rhodecode.lib.hooks.log_push_action'
+
+        hooks4 = RhodeCodeUi()
+        hooks4.ui_section = 'hooks'
+        hooks4.ui_key = 'preoutgoing.pull_logger'
+        hooks4.ui_value = 'python:rhodecode.lib.hooks.log_pull_action'
+
+        #For mercurial 1.7 set backward comapatibility with format
+        dotencode_disable = RhodeCodeUi()
+        dotencode_disable.ui_section = 'format'
+        dotencode_disable.ui_key = 'dotencode'
+        dotencode_disable.ui_value = 'false'
+
+        try:
+            self.sa.add(hooks1)
+            self.sa.add(hooks2)
+            self.sa.add(hooks3)
+            self.sa.add(hooks4)
+            self.sa.add(dotencode_disable)
+            self.sa.commit()
+        except:
+            self.sa.rollback()
+            raise
+
+
+    def create_ldap_options(self):
+        """Creates ldap settings"""
+
+        try:
+            for k in ['ldap_active', 'ldap_host', 'ldap_port', 'ldap_ldaps',
+                      'ldap_dn_user', 'ldap_dn_pass', 'ldap_base_dn']:
+
+                setting = RhodeCodeSettings(k, '')
+                self.sa.add(setting)
+            self.sa.commit()
+        except:
+            self.sa.rollback()
+            raise
     
     def config_prompt(self, test_repo_path=''):
         log.info('Setting up repositories config')
@@ -128,17 +329,9 class DbManage(object):
             log.error('You entered wrong path: %s', path)
             sys.exit()
         
-        hooks1 = RhodeCodeUi()
-        hooks1.ui_section = 'hooks'
-        hooks1.ui_key = 'changegroup.update'
-        hooks1.ui_value = 'hg update >&2'
-        hooks1.ui_active = False
+        self.create_ui_settings()
         
-        hooks2 = RhodeCodeUi()
-        hooks2.ui_section = 'hooks'
-        hooks2.ui_key = 'changegroup.repo_size'
-        hooks2.ui_value = 'python:rhodecode.lib.hooks.repo_size' 
-                
+        #HG UI OPTIONS
         web1 = RhodeCodeUi()
         web1.ui_section = 'web'
         web1.ui_key = 'push_ssl'
@@ -162,21 +355,14 class DbManage(object):
         paths = RhodeCodeUi()
         paths.ui_section = 'paths'
         paths.ui_key = '/'
-        paths.ui_value = os.path.join(path, '*')
+        paths.ui_value = path
         
         
-        hgsettings1 = RhodeCodeSettings()
+        hgsettings1 = RhodeCodeSettings('realm', 'RhodeCode authentication')
+        hgsettings2 = RhodeCodeSettings('title', 'RhodeCode')
         
-        hgsettings1.app_settings_name = 'realm'
-        hgsettings1.app_settings_value = 'RhodeCode authentication'
-        
-        hgsettings2 = RhodeCodeSettings()
-        hgsettings2.app_settings_name = 'title'
-        hgsettings2.app_settings_value = 'RhodeCode'      
         
         try:
-            self.sa.add(hooks1)
-            self.sa.add(hooks2)
             self.sa.add(web1)
             self.sa.add(web2)
             self.sa.add(web3)
@@ -184,10 +370,14 class DbManage(object):
             self.sa.add(paths)
             self.sa.add(hgsettings1)
             self.sa.add(hgsettings2)
+
             self.sa.commit()
         except:
             self.sa.rollback()
             raise        
+
+        self.create_ldap_options()
+
         log.info('created ui config')
                     
     def create_user(self, username, password, email='', admin=False):
@@ -214,9 +404,9 class DbManage(object):
         def_user = User()
         def_user.username = 'default'
         def_user.password = get_crypt_password(str(uuid.uuid1())[:8])
-        def_user.name = 'default'
-        def_user.lastname = 'default'
-        def_user.email = '[e-mail]'
+        def_user.name = 'Anonymous'
+        def_user.lastname = 'User'
+        def_user.email = '[e-mail]'
         def_user.admin = False
         def_user.active = False
         try:

rhodecode/lib/helpers.py

@@ -3,6 +3,8
 Consists of functions to typically be used within templates, but also
 available to Controllers. This module is available to both as 'h'.
 """
+import random
+import hashlib
 from pygments.formatters import HtmlFormatter
 from pygments import highlight as code_highlight
 from pylons import url, app_globals as g
@@ -23,6 +25,36 from webhelpers.pylonslib.secure_form im
 from webhelpers.text import chop_at, collapse, convert_accented_entities, \
     convert_misc_entities, lchop, plural, rchop, remove_formatting, \
     replace_whitespace, urlify, truncate, wrap_paragraphs
+from webhelpers.date import time_ago_in_words
+
+from webhelpers.html.tags import _set_input_attrs, _set_id_attr, \
+    convert_boolean_attrs, NotGiven
+
+def _reset(name, value=None, id=NotGiven, type="reset", **attrs):
+    _set_input_attrs(attrs, type, name, value)
+    _set_id_attr(attrs, id, name)
+    convert_boolean_attrs(attrs, ["disabled"])
+    return HTML.input(**attrs)
+
+reset = _reset
+
+
+def get_token():
+    """Return the current authentication token, creating one if one doesn't
+    already exist.
+    """
+    token_key = "_authentication_token"
+    from pylons import session
+    if not token_key in session:
+        try:
+            token = hashlib.sha1(str(random.getrandbits(128))).hexdigest()
+        except AttributeError: # Python < 2.4
+            token = hashlib.sha1(str(random.randrange(2 ** 128))).hexdigest()
+        session[token_key] = token
+        if hasattr(session, 'save'):
+            session.save()
+    return session[token_key]
+
 
 #Custom helpers here :)
 class _Link(object):
@@ -93,7 +125,7 class _ToolTip(object):
                 var tts = YAHOO.util.Dom.getElementsByClassName('tooltip');
                 
                 for (var i = 0; i < tts.length; i++) {
-                    //if element doesn not have and id autgenerate one for tooltip
+                    //if element doesn't not have and id autgenerate one for tooltip
                     
                     if (!tts[i].id){
                         tts[i].id='tt'+i*100;
@@ -111,7 +143,7 class _ToolTip(object):
                 showdelay:20,
             });
             
-            //Mouse Over event disabled for new repositories since they dont
+            //Mouse Over event disabled for new repositories since they don't
             //have last commit message
             myToolTips.contextMouseOverEvent.subscribe(
                 function(type, args) {
@@ -270,13 +302,13 def pygmentize_annotation(filenode, **kw
         tooltip_html = tooltip_html % (changeset.author,
                                                changeset.date,
                                                tooltip(changeset.message))
-        lnk_format = 'r%-5s:%s' % (changeset.revision,
-                                 changeset.short_id)
+        lnk_format = '%5s:%s' % ('r%s' % changeset.revision,
+                                 short_id(changeset.raw_id))
         uri = link_to(
                 lnk_format,
                 url('changeset_home', repo_name=changeset.repository.name,
-                    revision=changeset.short_id),
-                style=get_color_string(changeset.short_id),
+                    revision=changeset.raw_id),
+                style=get_color_string(changeset.raw_id),
                 class_='tooltip',
                 tooltip_title=tooltip_html
               )
@@ -317,37 +349,168 def get_changeset_safe(repo, rev):
 flash = _Flash()
 
 
-#===============================================================================
+#==============================================================================
 # MERCURIAL FILTERS available via h.
-#===============================================================================
+#==============================================================================
 from mercurial import util
-from mercurial.templatefilters import age as _age, person as _person
+from mercurial.templatefilters import person as _person
+
+
+
+def _age(curdate):
+    """turns a datetime into an age string."""
+
+    if not curdate:
+        return ''
+
+    from datetime import timedelta, datetime
+
+    agescales = [("year", 3600 * 24 * 365),
+                 ("month", 3600 * 24 * 30),
+                 ("day", 3600 * 24),
+                 ("hour", 3600),
+                 ("minute", 60),
+                 ("second", 1), ]
+
+    age = datetime.now() - curdate
+    age_seconds = (age.days * agescales[2][1]) + age.seconds
+    pos = 1
+    for scale in agescales:
+        if scale[1] <= age_seconds:
+            if pos == 6:pos = 5
+            return time_ago_in_words(curdate, agescales[pos][0]) + ' ' + _('ago')
+        pos += 1
+
+    return _('just now')
 
 age = lambda  x:_age(x)
 capitalize = lambda x: x.capitalize()
-date = lambda x: util.datestr(x)
 email = util.email
 email_or_none = lambda x: util.email(x) if util.email(x) != x else None
 person = lambda x: _person(x)
-hgdate = lambda  x: "%d %d" % x
-isodate = lambda  x: util.datestr(x, '%Y-%m-%d %H:%M %1%2')
-isodatesec = lambda  x: util.datestr(x, '%Y-%m-%d %H:%M:%S %1%2')
-localdate = lambda  x: (x[0], util.makedate()[1])
-rfc822date = lambda  x: util.datestr(x, "%a, %d %b %Y %H:%M:%S %1%2")
-rfc822date_notz = lambda  x: util.datestr(x, "%a, %d %b %Y %H:%M:%S")
-rfc3339date = lambda  x: util.datestr(x, "%Y-%m-%dT%H:%M:%S%1:%2")
-time_ago = lambda x: util.datestr(_age(x), "%a, %d %b %Y %H:%M:%S %1%2")
+short_id = lambda x: x[:12]
+
+
+def bool2icon(value):
+    """
+    Returns True/False values represented as small html image of true/false
+    icons
+    :param value: bool value
+    """
+
+    if value is True:
+        return HTML.tag('img', src="/images/icons/accept.png", alt=_('True'))
+
+    if value is False:
+        return HTML.tag('img', src="/images/icons/cancel.png", alt=_('False'))
+
+    return value
 
 
-#===============================================================================
+def action_parser(user_log):
+    """
+    This helper will map the specified string action into translated
+    fancy names with icons and links
+    
+    @param action:
+    """
+    action = user_log.action
+    action_params = ' '
+
+    x = action.split(':')
+
+    if len(x) > 1:
+        action, action_params = x
+
+    def get_cs_links():
+        if action == 'push':
+            revs_limit = 5
+            revs = action_params.split(',')
+            cs_links = " " + ', '.join ([link(rev,
+                    url('changeset_home',
+                    repo_name=user_log.repository.repo_name,
+                    revision=rev)) for rev in revs[:revs_limit] ])
+            if len(revs) > revs_limit:
+                uniq_id = revs[0]
+                html_tmpl = ('<span> %s '
+                '<a class="show_more" id="_%s" href="#">%s</a> '
+                '%s</span>')
+                cs_links += html_tmpl % (_('and'), uniq_id, _('%s more') \
+                                            % (len(revs) - revs_limit),
+                                            _('revisions'))
+
+                html_tmpl = '<span id="%s" style="display:none"> %s </span>'
+                cs_links += html_tmpl % (uniq_id, ', '.join([link(rev,
+                    url('changeset_home',
+                    repo_name=user_log.repository.repo_name,
+                    revision=rev)) for rev in revs[:revs_limit] ]))
+
+            return cs_links
+        return ''
+
+    def get_fork_name():
+        if action == 'user_forked_repo':
+            from rhodecode.model.scm import ScmModel
+            repo_name = action_params
+            repo = ScmModel().get(repo_name)
+            if repo is None:
+                return repo_name
+            return link_to(action_params, url('summary_home',
+                                              repo_name=repo.name,),
+                                              title=repo.dbrepo.description)
+        return ''
+    map = {'user_deleted_repo':_('User [deleted] repository'),
+           'user_created_repo':_('User [created] repository'),
+           'user_forked_repo':_('User [forked] repository as: %s') % get_fork_name(),
+           'user_updated_repo':_('User [updated] repository'),
+           'admin_deleted_repo':_('Admin [delete] repository'),
+           'admin_created_repo':_('Admin [created] repository'),
+           'admin_forked_repo':_('Admin [forked] repository'),
+           'admin_updated_repo':_('Admin [updated] repository'),
+           'push':_('[Pushed] %s') % get_cs_links(),
+           'pull':_('[Pulled]'),
+           'started_following_repo':_('User [started following] repository'),
+           'stopped_following_repo':_('User [stopped following] repository'),
+            }
+
+    action_str = map.get(action, action)
+    return literal(action_str.replace('[', '<span class="journal_highlight">')\
+                   .replace(']', '</span>'))
+
+def action_parser_icon(user_log):
+    action = user_log.action
+    action_params = None
+    x = action.split(':')
+
+    if len(x) > 1:
+        action, action_params = x
+
+    tmpl = """<img src="/images/icons/%s" alt="%s"/>"""
+    map = {'user_deleted_repo':'database_delete.png',
+           'user_created_repo':'database_add.png',
+           'user_forked_repo':'arrow_divide.png',
+           'user_updated_repo':'database_edit.png',
+           'admin_deleted_repo':'database_delete.png',
+           'admin_created_repo':'database_ddd.png',
+           'admin_forked_repo':'arrow_divide.png',
+           'admin_updated_repo':'database_edit.png',
+           'push':'script_add.png',
+           'pull':'down_16.png',
+           'started_following_repo':'heart_add.png',
+           'stopped_following_repo':'heart_delete.png',
+            }
+    return literal(tmpl % (map.get(action, action), action))
+
+
+#==============================================================================
 # PERMS
-#===============================================================================
+#==============================================================================
 from rhodecode.lib.auth import HasPermissionAny, HasPermissionAll, \
 HasRepoPermissionAny, HasRepoPermissionAll
 
-#===============================================================================
+#==============================================================================
 # GRAVATAR URL
-#===============================================================================
+#==============================================================================
 import hashlib
 import urllib
 from pylons import request

rhodecode/lib/hooks.py

@@ -22,12 +22,12 Created on Aug 6, 2010
 
 @author: marcink
 """
-
-import sys
+from mercurial.cmdutil import revrange
+from mercurial.node import nullrev
+from rhodecode.lib import helpers as h
+from rhodecode.lib.utils import action_logger
 import os
-from rhodecode.lib import helpers as h
-from rhodecode.model import meta
-from rhodecode.model.db import UserLog, User
+import sys
 
 def repo_size(ui, repo, hooktype=None, **kwargs):
 
@@ -54,31 +54,52 def repo_size(ui, repo, hooktype=None, *
     sys.stdout.write('Repository size .hg:%s repo:%s total:%s\n' \
                      % (size_hg_f, size_root_f, size_total_f))
     
-    user_action_mapper(ui, repo, hooktype, **kwargs)
+def log_pull_action(ui, repo, **kwargs):
+    """
+    Logs user last pull action
+    :param ui:
+    :param repo:
+    """
 
-def user_action_mapper(ui, repo, hooktype=None, **kwargs):
+    extra_params = dict(repo.ui.configitems('rhodecode_extras'))
+    username = extra_params['username']
+    repository = extra_params['repository']
+    action = 'pull'
+
+    action_logger(username, action, repository, extra_params['ip'])
+
+    return 0
+
+def log_push_action(ui, repo, **kwargs):
     """
     Maps user last push action to new changeset id, from mercurial
     :param ui:
     :param repo:
-    :param hooktype:
     """
     
-    try:
-        sa = meta.Session
-        username = kwargs['url'].split(':')[-1]
-        user_log = sa.query(UserLog)\
-            .filter(UserLog.user == sa.query(User)\
-                                    .filter(User.username == username).one())\
-            .order_by(UserLog.user_log_id.desc()).first()
+    extra_params = dict(repo.ui.configitems('rhodecode_extras'))
+    username = extra_params['username']
+    repository = extra_params['repository']
+    action = 'push:%s'
+    node = kwargs['node']
+
+    def get_revs(repo, rev_opt):
+        if rev_opt:
+            revs = revrange(repo, rev_opt)
         
-        if user_log and not user_log.revision:
-            user_log.revision = str(repo['tip'])
-            sa.add(user_log)
-            sa.commit()
+            if len(revs) == 0:
+                return (nullrev, nullrev)
+            return (max(revs), min(revs))
+        else:
+            return (len(repo) - 1, 0)
+
+    stop, start = get_revs(repo, [node + ':'])
         
-    except Exception, e:
-        sa.rollback()
-        raise
-    finally:
-        meta.Session.remove()    
+    revs = (str(repo[r]) for r in xrange(start, stop + 1))
+
+    action = action % ','.join(revs)
+
+    action_logger(username, action, repository, extra_params['ip'])
+
+    return 0
+

rhodecode/lib/indexers/__init__.py

@@ -1,26 +1,28
+import os
+import sys
+import traceback
 from os.path import dirname as dn, join as jn
+
+#to get the rhodecode import
+sys.path.append(dn(dn(dn(os.path.realpath(__file__)))))
+
+from rhodecode.model import init_model
+from rhodecode.model.scm import ScmModel
 from rhodecode.config.environment import load_environment
-from rhodecode.model.hg_model import HgModel
+from rhodecode.lib.utils import BasePasterCommand, Command, add_cache
+
 from shutil import rmtree
 from webhelpers.html.builder import escape
 from vcs.utils.lazy import LazyProperty
 
+from sqlalchemy import engine_from_config
+
 from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter
 from whoosh.fields import TEXT, ID, STORED, Schema, FieldType
 from whoosh.index import create_in, open_dir
 from whoosh.formats import Characters
 from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter   
 
-import os
-import sys
-import traceback
-
-#to get the rhodecode import
-sys.path.append(dn(dn(dn(os.path.realpath(__file__)))))
-
-
-#LOCATION WE KEEP THE INDEX
-IDX_LOCATION = jn(dn(dn(dn(dn(os.path.abspath(__file__))))), 'data', 'index')
 
 #EXTENSIONS WE WANT TO INDEX CONTENT OFF
 INDEX_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c',
@@ -48,6 +50,55 IDX_NAME = 'HG_INDEX'
 FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n') 
 FRAGMENTER = SimpleFragmenter(200)
                             
+
+class MakeIndex(BasePasterCommand):
+
+    max_args = 1
+    min_args = 1
+
+    usage = "CONFIG_FILE"
+    summary = "Creates index for full text search given configuration file"
+    group_name = "RhodeCode"
+    takes_config_file = -1
+    parser = Command.standard_parser(verbose=True)
+
+    def command(self):
+
+        from pylons import config
+        add_cache(config)
+        engine = engine_from_config(config, 'sqlalchemy.db1.')
+        init_model(engine)
+
+        index_location = config['index_dir']
+        repo_location = self.options.repo_location
+
+        #======================================================================
+        # WHOOSH DAEMON
+        #======================================================================
+        from rhodecode.lib.pidlock import LockHeld, DaemonLock
+        from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
+        try:
+            l = DaemonLock()
+            WhooshIndexingDaemon(index_location=index_location,
+                                 repo_location=repo_location)\
+                .run(full_index=self.options.full_index)
+            l.release()
+        except LockHeld:
+            sys.exit(1)
+
+    def update_parser(self):
+        self.parser.add_option('--repo-location',
+                          action='store',
+                          dest='repo_location',
+                          help="Specifies repositories location to index REQUIRED",
+                          )
+        self.parser.add_option('-f',
+                          action='store_true',
+                          dest='full_index',
+                          help="Specifies that index should be made full i.e"
+                                " destroy old and build from scratch",
+                          default=False)
+
 class ResultWrapper(object):
     def __init__(self, search_type, searcher, matcher, highlight_items):
         self.search_type = search_type
@@ -115,8 +166,8 class ResultWrapper(object):
         Smart function that implements chunking the content
         but not overlap chunks so it doesn't highlight the same
         close occurrences twice.
-        :param matcher:
-        :param size:
+        @param matcher:
+        @param size:
         """
         memory = [(0, 0)]
         for span in self.matcher.spans():

rhodecode/lib/indexers/daemon.py

@@ -32,12 +32,12 from os.path import join as jn
 project_path = dn(dn(dn(dn(os.path.realpath(__file__)))))
 sys.path.append(project_path)
 
-from rhodecode.lib.pidlock import LockHeld, DaemonLock
-from rhodecode.model.hg_model import HgModel
+
+from rhodecode.model.scm import ScmModel
 from rhodecode.lib.helpers import safe_unicode
 from whoosh.index import create_in, open_dir
 from shutil import rmtree
-from rhodecode.lib.indexers import INDEX_EXTENSIONS, IDX_LOCATION, SCHEMA, IDX_NAME
+from rhodecode.lib.indexers import INDEX_EXTENSIONS, SCHEMA, IDX_NAME
 
 from time import mktime
 from vcs.exceptions import ChangesetError, RepositoryError
@@ -61,35 +61,38 ch.setFormatter(formatter)
 # add ch to logger
 log.addHandler(ch)
 
-def scan_paths(root_location):
-    return HgModel.repo_scan('/', root_location, None, True)
-
 class WhooshIndexingDaemon(object):
     """
     Deamon for atomic jobs
     """
 
-    def __init__(self, indexname='HG_INDEX', repo_location=None):
+    def __init__(self, indexname='HG_INDEX', index_location=None,
+                 repo_location=None, sa=None):
         self.indexname = indexname
+
+        self.index_location = index_location
+        if not index_location:
+            raise Exception('You have to provide index location')
+
         self.repo_location = repo_location
-        self.repo_paths = scan_paths(self.repo_location)
+        if not repo_location:
+            raise Exception('You have to provide repositories location')
+
+        self.repo_paths = ScmModel(sa).repo_scan(self.repo_location, None)
         self.initial = False
-        if not os.path.isdir(IDX_LOCATION):
-            os.mkdir(IDX_LOCATION)
+        if not os.path.isdir(self.index_location):
+            os.makedirs(self.index_location)
             log.info('Cannot run incremental index since it does not'
                      ' yet exist running full build')
             self.initial = True
         
     def get_paths(self, repo):
-        """
-        recursive walk in root dir and return a set of all path in that dir
+        """recursive walk in root dir and return a set of all path in that dir
         based on repository walk function
         """
         index_paths_ = set()
         try:
-            tip = repo.get_changeset()
-            
-            for topnode, dirs, files in tip.walk('/'):
+            for topnode, dirs, files in repo.walk('/', 'tip'):
                 for f in files:
                     index_paths_.add(jn(repo.path, f.path))
                 for dir in dirs:
@@ -109,7 +112,8 class WhooshIndexingDaemon(object):
         return mktime(node.last_changeset.date.timetuple())
     
     def add_doc(self, writer, path, repo):
-        """Adding doc to writer"""
+        """Adding doc to writer this function itself fetches data from
+        the instance of vcs backend"""
         node = self.get_node(repo, path)
 
         #we just index the content of chosen files
@@ -130,14 +134,14 class WhooshIndexingDaemon(object):
 
     
     def build_index(self):
-        if os.path.exists(IDX_LOCATION):
+        if os.path.exists(self.index_location):
             log.debug('removing previous index')
-            rmtree(IDX_LOCATION)
+            rmtree(self.index_location)
             
-        if not os.path.exists(IDX_LOCATION):
-            os.mkdir(IDX_LOCATION)
+        if not os.path.exists(self.index_location):
+            os.mkdir(self.index_location)
         
-        idx = create_in(IDX_LOCATION, SCHEMA, indexname=IDX_NAME)
+        idx = create_in(self.index_location, SCHEMA, indexname=IDX_NAME)
         writer = idx.writer()
         
         for cnt, repo in enumerate(self.repo_paths.values()):
@@ -154,7 +158,7 class WhooshIndexingDaemon(object):
     def update_index(self):
         log.debug('STARTING INCREMENTAL INDEXING UPDATE')
             
-        idx = open_dir(IDX_LOCATION, indexname=self.indexname)
+        idx = open_dir(self.index_location, indexname=self.indexname)
         # The set of all paths in the index
         indexed_paths = set()
         # The set of all paths we need to re-index
@@ -209,40 +213,3 class WhooshIndexingDaemon(object):
             self.build_index()
         else:
             self.update_index()
-        
-if __name__ == "__main__":
-    arg = sys.argv[1:]
-    if len(arg) != 2:
-        sys.stderr.write('Please specify indexing type [full|incremental]' 
-                         'and path to repositories as script args \n')
-        sys.exit()
-    
-    
-    if arg[0] == 'full':
-        full_index = True
-    elif arg[0] == 'incremental':
-        # False means looking just for changes
-        full_index = False
-    else:
-        sys.stdout.write('Please use [full|incremental]' 
-                         ' as script first arg \n')
-        sys.exit()
-    
-    if not os.path.isdir(arg[1]):
-        sys.stderr.write('%s is not a valid path \n' % arg[1])
-        sys.exit()
-    else:
-        if arg[1].endswith('/'):
-            repo_location = arg[1] + '*'
-        else:
-            repo_location = arg[1] + '/*'
-    
-    try:
-        l = DaemonLock()
-        WhooshIndexingDaemon(repo_location=repo_location)\
-            .run(full_index=full_index)
-        l.release()
-        reload(logging)
-    except LockHeld:
-        sys.exit(1)
-

rhodecode/lib/middleware/simplegit.py

@@ -17,6 +17,14
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
+"""
+Created on 2010-04-28
+
+@author: marcink
+SimpleGit middleware for handling git protocol request (push/clone etc.)
+It's implemented with basic auth function
+"""
+
 from dulwich import server as dulserver
 
 class SimpleGitUploadPackHandler(dulserver.UploadPackHandler):
@@ -54,26 +62,28 from dulwich.repo import Repo
 from dulwich.web import HTTPGitApplication
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
-from rhodecode.lib.auth import authfunc, HasPermissionAnyMiddleware, \
-    get_user_cached
-from rhodecode.lib.utils import action_logger, is_git, invalidate_cache, \
-    check_repo_fast
+from rhodecode.lib.auth import authfunc, HasPermissionAnyMiddleware
+from rhodecode.lib.utils import invalidate_cache, check_repo_fast
+from rhodecode.model.user import UserModel
 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError
 import logging
 import os
 import traceback
-"""
-Created on 2010-04-28
-
-@author: marcink
-SimpleGit middleware for handling git protocol request (push/clone etc.)
-It's implemented with basic auth function
-"""
-
-
 
 log = logging.getLogger(__name__)
 
+def is_git(environ):
+    """
+    Returns True if request's target is git server. ``HTTP_USER_AGENT`` would
+    then have git client version given.
+    
+    :param environ:
+    """
+    http_user_agent = environ.get('HTTP_USER_AGENT')
+    if http_user_agent and http_user_agent.startswith('git'):
+        return True
+    return False
+
 class SimpleGit(object):
 
     def __init__(self, application, config):
@@ -81,11 +91,19 class SimpleGit(object):
         self.config = config
         #authenticate this git request using 
         self.authenticate = AuthBasicAuthenticator('', authfunc)
+        self.ipaddr = '0.0.0.0'
+        self.repository = None
+        self.username = None
+        self.action = None
 
     def __call__(self, environ, start_response):
         if not is_git(environ):
             return self.application(environ, start_response)
 
+        proxy_key = 'HTTP_X_REAL_IP'
+        def_key = 'REMOTE_ADDR'
+        self.ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
+
         #===================================================================
         # AUTHENTICATE THIS GIT REQUEST
         #===================================================================
@@ -99,10 +117,14 class SimpleGit(object):
             else:
                 return result.wsgi_application(environ, start_response)
 
+        #=======================================================================
+        # GET REPOSITORY
+        #=======================================================================
         try:
-            self.repo_name = environ['PATH_INFO'].split('/')[1]
-            if self.repo_name.endswith('/'):
-                self.repo_name = self.repo_name.rstrip('/')
+            repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
+            if repo_name.endswith('/'):
+                repo_name = repo_name.rstrip('/')
+            self.repository = repo_name
         except:
             log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
@@ -110,20 +132,21 class SimpleGit(object):
         #===================================================================
         # CHECK PERMISSIONS FOR THIS REQUEST
         #===================================================================
-        action = self.__get_action(environ)
-        if action:
+        self.action = self.__get_action(environ)
+        if self.action:
             username = self.__get_environ_user(environ)
             try:
                 user = self.__get_user(username)
+                self.username = user.username
             except:
                 log.error(traceback.format_exc())
                 return HTTPInternalServerError()(environ, start_response)
 
             #check permissions for this repository
-            if action == 'push':
+            if self.action == 'push':
                 if not HasPermissionAnyMiddleware('repository.write',
                                                   'repository.admin')\
-                                                    (user, self.repo_name):
+                                                    (user, repo_name):
                     return HTTPForbidden()(environ, start_response)
 
             else:
@@ -131,15 +154,13 class SimpleGit(object):
                 if not HasPermissionAnyMiddleware('repository.read',
                                                   'repository.write',
                                                   'repository.admin')\
-                                                    (user, self.repo_name):
+                                                    (user, repo_name):
                     return HTTPForbidden()(environ, start_response)
 
-            #log action
-            if action in ('push', 'pull', 'clone'):
-                proxy_key = 'HTTP_X_REAL_IP'
-                def_key = 'REMOTE_ADDR'
-                ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
-                self.__log_user_action(user, action, self.repo_name, ipaddr)
+        self.extras = {'ip':self.ipaddr,
+                       'username':self.username,
+                       'action':self.action,
+                       'repository':self.repository}
 
         #===================================================================
         # GIT REQUEST HANDLING
@@ -151,12 +172,12 class SimpleGit(object):
             return HTTPNotFound()(environ, start_response)
         try:
             app = self.__make_app()
-        except Exception:
+        except:
             log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
 
         #invalidate cache on push
-        if action == 'push':
+        if self.action == 'push':
             self.__invalidate_cache(self.repo_name)
             messages = []
             messages.append('thank you for using rhodecode')
@@ -175,7 +196,7 class SimpleGit(object):
         return environ.get('REMOTE_USER')
 
     def __get_user(self, username):
-        return get_user_cached(username)
+        return UserModel().get_by_username(username, cache=True)
 
     def __get_action(self, environ):
         """
@@ -193,12 +214,8 class SimpleGit(object):
         else:
             return 'other'
 
-    def __log_user_action(self, user, action, repo, ipaddr):
-        action_logger(user, action, repo, ipaddr)
-
     def __invalidate_cache(self, repo_name):
         """we know that some change was made to repositories and we should
         invalidate the cache to see the changes right away but only for
         push requests"""
-        invalidate_cache('cached_repo_list')
-        invalidate_cache('full_changelog', repo_name)
+        invalidate_cache('get_repo_cached_%s' % repo_name)

rhodecode/lib/middleware/simplehg.py

@@ -24,40 +24,57 Created on 2010-04-28
 SimpleHG middleware for handling mercurial protocol request (push/clone etc.)
 It's implemented with basic auth function
 """
-from itertools import chain
 from mercurial.error import RepoError
 from mercurial.hgweb import hgweb
 from mercurial.hgweb.request import wsgiapplication
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
-from rhodecode.lib.auth import authfunc, HasPermissionAnyMiddleware, \
-    get_user_cached
-from rhodecode.lib.utils import is_mercurial, make_ui, invalidate_cache, \
+from rhodecode.lib.auth import authfunc, HasPermissionAnyMiddleware
+from rhodecode.lib.utils import make_ui, invalidate_cache, \
     check_repo_fast, ui_sections
+from rhodecode.model.user import UserModel
 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError
-from rhodecode.lib.utils import action_logger
 import logging
 import os
 import traceback
 
 log = logging.getLogger(__name__)
 
+def is_mercurial(environ):
+    """
+    Returns True if request's target is mercurial server - header
+    ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
+    """
+    http_accept = environ.get('HTTP_ACCEPT')
+    if http_accept and http_accept.startswith('application/mercurial'):
+        return True
+    return False
+
 class SimpleHg(object):
 
     def __init__(self, application, config):
         self.application = application
         self.config = config
-        #authenticate this mercurial request using 
+        #authenticate this mercurial request using authfunc
         self.authenticate = AuthBasicAuthenticator('', authfunc)
+        self.ipaddr = '0.0.0.0'
+        self.repository = None
+        self.username = None
+        self.action = None
 
     def __call__(self, environ, start_response):
         if not is_mercurial(environ):
             return self.application(environ, start_response)
 
+        proxy_key = 'HTTP_X_REAL_IP'
+        def_key = 'REMOTE_ADDR'
+        self.ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
+
         #===================================================================
         # AUTHENTICATE THIS MERCURIAL REQUEST
         #===================================================================
         username = REMOTE_USER(environ)
+
         if not username:
             self.authenticate.realm = self.config['rhodecode_realm']
             result = self.authenticate(environ)
@@ -67,10 +84,14 class SimpleHg(object):
             else:
                 return result.wsgi_application(environ, start_response)
 
+        #=======================================================================
+        # GET REPOSITORY
+        #=======================================================================
         try:
             repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
             if repo_name.endswith('/'):
                 repo_name = repo_name.rstrip('/')
+            self.repository = repo_name
         except:
             log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
@@ -78,17 +99,18 class SimpleHg(object):
         #===================================================================
         # CHECK PERMISSIONS FOR THIS REQUEST
         #===================================================================
-        action = self.__get_action(environ)
-        if action:
+        self.action = self.__get_action(environ)
+        if self.action:
             username = self.__get_environ_user(environ)
             try:
                 user = self.__get_user(username)
+                self.username = user.username
             except:
                 log.error(traceback.format_exc())
                 return HTTPInternalServerError()(environ, start_response)
 
             #check permissions for this repository
-            if action == 'push':
+            if self.action == 'push':
                 if not HasPermissionAnyMiddleware('repository.write',
                                                   'repository.admin')\
                                                     (user, repo_name):
@@ -102,12 +124,10 class SimpleHg(object):
                                                     (user, repo_name):
                     return HTTPForbidden()(environ, start_response)
 
-            #log action
-            if action in ('push', 'pull', 'clone'):
-                proxy_key = 'HTTP_X_REAL_IP'
-                def_key = 'REMOTE_ADDR'
-                ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
-                self.__log_user_action(user, action, repo_name, ipaddr)
+        self.extras = {'ip':self.ipaddr,
+                       'username':self.username,
+                       'action':self.action,
+                       'repository':self.repository}
 
         #===================================================================
         # MERCURIAL REQUEST HANDLING
@@ -130,40 +150,21 class SimpleHg(object):
             return HTTPInternalServerError()(environ, start_response)
 
         #invalidate cache on push
-        if action == 'push':
+        if self.action == 'push':
             self.__invalidate_cache(repo_name)
-            messages = []
-            messages.append('thank you for using rhodecode')
 
-            return self.msg_wrapper(app, environ, start_response, messages)
-        else:
             return app(environ, start_response)
 
 
-    def msg_wrapper(self, app, environ, start_response, messages=[]):
-        """
-        Wrapper for custom messages that come out of mercurial respond messages
-        is a list of messages that the user will see at the end of response 
-        from merurial protocol actions that involves remote answers
-        :param app:
-        :param environ:
-        :param start_response:
-        """
-        def custom_messages(msg_list):
-            for msg in msg_list:
-                yield msg + '\n'
-        org_response = app(environ, start_response)
-        return chain(org_response, custom_messages(messages))
-
     def __make_app(self):
         hgserve = hgweb(str(self.repo_path), baseui=self.baseui)
-        return  self.__load_web_settings(hgserve)
+        return  self.__load_web_settings(hgserve, self.extras)
 
     def __get_environ_user(self, environ):
         return environ.get('REMOTE_USER')
 
     def __get_user(self, username):
-        return get_user_cached(username)
+        return UserModel().get_by_username(username, cache=True)
 
     def __get_action(self, environ):
         """
@@ -174,7 +175,7 class SimpleHg(object):
         mapping = {'changegroup': 'pull',
                    'changegroupsubset': 'pull',
                    'stream_out': 'pull',
-                   #'listkeys': 'pull',
+                   'listkeys': 'pull',
                    'unbundle': 'push',
                    'pushkey': 'push', }
         for qry in environ['QUERY_STRING'].split('&'):
@@ -185,25 +186,26 class SimpleHg(object):
                 else:
                     return cmd
 
-    def __log_user_action(self, user, action, repo, ipaddr):
-        action_logger(user, action, repo, ipaddr)
-
     def __invalidate_cache(self, repo_name):
         """we know that some change was made to repositories and we should
         invalidate the cache to see the changes right away but only for
         push requests"""
-        invalidate_cache('cached_repo_list')
-        invalidate_cache('full_changelog', repo_name)
+        invalidate_cache('get_repo_cached_%s' % repo_name)
 
 
-    def __load_web_settings(self, hgserve):
+    def __load_web_settings(self, hgserve, extras={}):
         #set the global ui for hgserve instance passed
         hgserve.repo.ui = self.baseui
 
         hgrc = os.path.join(self.repo_path, '.hg', 'hgrc')
+
+        #inject some additional parameters that will be available in ui
+        #for hooks
+        for k, v in extras.items():
+            hgserve.repo.ui.setconfig('rhodecode_extras', k, v)
+
         repoui = make_ui('file', hgrc, False)
 
-
         if repoui:
             #overwrite our ui instance with the section from hgrc file
             for section in ui_sections:

rhodecode/lib/smtp_mailer.py

@@ -47,7 +47,7 class SmtpMailer(object):
         if self.debug:    
             smtp_serv.set_debuglevel(1)
 
-        smtp_serv.ehlo("mailer")
+        smtp_serv.ehlo("rhodecode mailer")
 
         #if server requires authorization you must provide login and password
         smtp_serv.login(self.user, self.passwd)

rhodecode/lib/utils.py

@@ -1,7 +1,15
-#!/usr/bin/env python
-# encoding: utf-8
-# Utilities for RhodeCode
-# Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.lib.utils
+    ~~~~~~~~~~~~~~~~~~~
+
+    Utilities library for RhodeCode
+    
+    :created_on: Apr 18, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
@@ -17,21 +25,28
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
-"""
-Created on April 18, 2010
-Utilities for RhodeCode
-@author: marcink
-"""
-from beaker.cache import cache_region
+import os
+import logging
+import datetime
+import traceback
+
+from UserDict import DictMixin
+
 from mercurial import ui, config, hg
 from mercurial.error import RepoError
-from rhodecode.model import meta
-from rhodecode.model.db import Repository, User, RhodeCodeUi, RhodeCodeSettings, UserLog
+
+import paste
+import beaker
+from paste.script.command import Command, BadCommand
+
 from vcs.backends.base import BaseChangeset
 from vcs.utils.lazy import LazyProperty
-import logging
-import datetime
-import os
+
+from rhodecode.model import meta
+from rhodecode.model.caching_query import FromCache
+from rhodecode.model.db import Repository, User, RhodeCodeUi, UserLog
+from rhodecode.model.repo import RepoModel
+from rhodecode.model.user import UserModel
 
 log = logging.getLogger(__name__)
 
@@ -39,72 +54,95 log = logging.getLogger(__name__)
 def get_repo_slug(request):
     return request.environ['pylons.routes_dict'].get('repo_name')
 
-def is_mercurial(environ):
-    """
-    Returns True if request's target is mercurial server - header
-    ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
-    """
-    http_accept = environ.get('HTTP_ACCEPT')
-    if http_accept and http_accept.startswith('application/mercurial'):
-        return True
-    return False
-
-def is_git(environ):
+def action_logger(user, action, repo, ipaddr='', sa=None):
     """
-    Returns True if request's target is git server. ``HTTP_USER_AGENT`` would
-    then have git client version given.
+    Action logger for various actions made by users
     
-    :param environ:
-    """
-    http_user_agent = environ.get('HTTP_USER_AGENT')
-    if http_user_agent.startswith('git'):
-        return True
-    return False
+    :param user: user that made this action, can be a unique username string or
+        object containing user_id attribute
+    :param action: action to log, should be on of predefined unique actions for
+        easy translations
+    :param repo: string name of repository or object containing repo_id,
+        that action was made on
+    :param ipaddr: optional ip address from what the action was made
+    :param sa: optional sqlalchemy session
 
-def action_logger(user, action, repo, ipaddr, sa=None):
-    """
-    Action logger for various action made by users
     """
 
     if not sa:
-        sa = meta.Session
+        sa = meta.Session()
 
     try:
+        um = UserModel()
         if hasattr(user, 'user_id'):
-            user_id = user.user_id
+            user_obj = user
         elif isinstance(user, basestring):
-            user_id = sa.query(User).filter(User.username == user).one()
+            user_obj = um.get_by_username(user, cache=False)
         else:
             raise Exception('You have to provide user object or username')
 
+
+        rm = RepoModel()
+        if hasattr(repo, 'repo_id'):
+            repo_obj = rm.get(repo.repo_id, cache=False)
+            repo_name = repo_obj.repo_name
+        elif  isinstance(repo, basestring):
         repo_name = repo.lstrip('/')
+            repo_obj = rm.get_by_repo_name(repo_name, cache=False)
+        else:
+            raise Exception('You have to provide repository to action logger')
+
+
         user_log = UserLog()
-        user_log.user_id = user_id
+        user_log.user_id = user_obj.user_id
         user_log.action = action
+
+        user_log.repository_id = repo_obj.repo_id
         user_log.repository_name = repo_name
-        user_log.repository = sa.query(Repository)\
-            .filter(Repository.repo_name == repo_name).one()
+
         user_log.action_date = datetime.datetime.now()
         user_log.user_ip = ipaddr
         sa.add(user_log)
         sa.commit()
 
-        log.info('Adding user %s, action %s on %s',
-                                        user.username, action, repo)
-    except Exception, e:
+        log.info('Adding user %s, action %s on %s', user_obj, action, repo)
+    except:
+        log.error(traceback.format_exc())
         sa.rollback()
-        log.error('could not log user action:%s', str(e))
 
-def check_repo_dir(paths):
-    repos_path = paths[0][1].split('/')
-    if repos_path[-1] in ['*', '**']:
-        repos_path = repos_path[:-1]
-    if repos_path[0] != '/':
-        repos_path[0] = '/'
-    if not os.path.isdir(os.path.join(*repos_path)):
-        raise Exception('Not a valid repository in %s' % paths[0][1])
+def get_repos(path, recursive=False, initial=False):
+    """
+    Scans given path for repos and return (name,(type,path)) tuple 
+    :param prefix:
+    :param path:
+    :param recursive:
+    :param initial:
+    """
+    from vcs.utils.helpers import get_scm
+    from vcs.exceptions import VCSError
+
+    try:
+        scm = get_scm(path)
+    except:
+        pass
+    else:
+        raise Exception('The given path %s should not be a repository got %s',
+                        path, scm)
+
+    for dirpath in os.listdir(path):
+        try:
+            yield dirpath, get_scm(os.path.join(path, dirpath))
+        except VCSError:
+            pass
 
 def check_repo_fast(repo_name, base_path):
+    """
+    Check given path for existance of directory
+    :param repo_name:
+    :param base_path:
+    
+    :return False: if this directory is present
+    """
     if os.path.isdir(os.path.join(base_path, repo_name)):return False
     return True
 
@@ -135,57 +173,6 def ask_ok(prompt, retries=4, complaint=
         if retries < 0: raise IOError
         print complaint
 
-@cache_region('super_short_term', 'cached_hg_ui')
-def get_hg_ui_cached():
-    try:
-        sa = meta.Session
-        ret = sa.query(RhodeCodeUi).all()
-    finally:
-        meta.Session.remove()
-    return ret
-
-
-def get_hg_settings():
-    try:
-        sa = meta.Session
-        ret = sa.query(RhodeCodeSettings).all()
-    finally:
-        meta.Session.remove()
-
-    if not ret:
-        raise Exception('Could not get application settings !')
-    settings = {}
-    for each in ret:
-        settings['rhodecode_' + each.app_settings_name] = each.app_settings_value
-
-    return settings
-
-def get_hg_ui_settings():
-    try:
-        sa = meta.Session
-        ret = sa.query(RhodeCodeUi).all()
-    finally:
-        meta.Session.remove()
-
-    if not ret:
-        raise Exception('Could not get application ui settings !')
-    settings = {}
-    for each in ret:
-        k = each.ui_key
-        v = each.ui_value
-        if k == '/':
-            k = 'root_path'
-
-        if k.find('.') != -1:
-            k = k.replace('.', '_')
-
-        if each.ui_section == 'hooks':
-            v = each.ui_active
-
-        settings[each.ui_section + '_' + k] = v
-
-    return settings
-
 #propagated from mercurial documentation
 ui_sections = ['alias', 'auth',
                 'decode/encode', 'defaults',
@@ -210,6 +197,11 def make_ui(read_from='file', path=None,
 
     baseui = ui.ui()
 
+    #clean the baseui object
+    baseui._ocfg = config.config()
+    baseui._ucfg = config.config()
+    baseui._tcfg = config.config()
+
     if read_from == 'file':
         if not os.path.isfile(path):
             log.warning('Unable to read config file %s' % path)
@@ -219,70 +211,69 def make_ui(read_from='file', path=None,
         cfg.read(path)
         for section in ui_sections:
             for k, v in cfg.items(section):
+                log.debug('settings ui from file[%s]%s:%s', section, k, v)
                 baseui.setconfig(section, k, v)
-                log.debug('settings ui from file[%s]%s:%s', section, k, v)
-
-        for k, v in baseui.configitems('extensions'):
-            baseui.setconfig('extensions', k, '0')
-        #just enable mq
-        baseui.setconfig('extensions', 'mq', '1')
-        if checkpaths:check_repo_dir(cfg.items('paths'))
 
 
     elif read_from == 'db':
-        hg_ui = get_hg_ui_cached()
+        sa = meta.Session()
+        ret = sa.query(RhodeCodeUi)\
+            .options(FromCache("sql_cache_short",
+                               "get_hg_ui_settings")).all()
+
+        hg_ui = ret
         for ui_ in hg_ui:
             if ui_.ui_active:
-                log.debug('settings ui from db[%s]%s:%s', ui_.ui_section, ui_.ui_key, ui_.ui_value)
+                log.debug('settings ui from db[%s]%s:%s', ui_.ui_section,
+                          ui_.ui_key, ui_.ui_value)
                 baseui.setconfig(ui_.ui_section, ui_.ui_key, ui_.ui_value)
 
-
+        meta.Session.remove()
     return baseui
 
 
 def set_rhodecode_config(config):
-    hgsettings = get_hg_settings()
+    """
+    Updates pylons config with new settings from database
+    :param config:
+    """
+    from rhodecode.model.settings import SettingsModel
+    hgsettings = SettingsModel().get_app_settings()
 
     for k, v in hgsettings.items():
         config[k] = v
 
-def invalidate_cache(name, *args):
-    """Invalidates given name cache"""
-
-    from beaker.cache import region_invalidate
-    log.info('INVALIDATING CACHE FOR %s', name)
+def invalidate_cache(cache_key, *args):
+    """
+    Puts cache invalidation task into db for 
+    further global cache invalidation
+    """
+    from rhodecode.model.scm import ScmModel
 
-    """propagate our arguments to make sure invalidation works. First
-    argument has to be the name of cached func name give to cache decorator
-    without that the invalidation would not work"""
-    tmp = [name]
-    tmp.extend(args)
-    args = tuple(tmp)
-
-    if name == 'cached_repo_list':
-        from rhodecode.model.hg_model import _get_repos_cached
-        region_invalidate(_get_repos_cached, None, *args)
-
-    if name == 'full_changelog':
-        from rhodecode.model.hg_model import _full_changelog_cached
-        region_invalidate(_full_changelog_cached, None, *args)
+    if cache_key.startswith('get_repo_cached_'):
+        name = cache_key.split('get_repo_cached_')[-1]
+        ScmModel().mark_for_invalidation(name)
 
 class EmptyChangeset(BaseChangeset):
     """
-    An dummy empty changeset.
+    An dummy empty changeset. It's possible to pass hash when creating
+    an EmptyChangeset
     """
 
-    revision = -1
-    message = ''
-    author = ''
-    date = ''
+    def __init__(self, cs='0' * 40):
+        self._empty_cs = cs
+        self.revision = -1
+        self.message = ''
+        self.author = ''
+        self.date = ''
+
     @LazyProperty
     def raw_id(self):
         """
-        Returns raw string identifing this changeset, useful for web
+        Returns raw string identifying this changeset, useful for web
         representation.
         """
-        return '0' * 40
+        return self._empty_cs
 
     @LazyProperty
     def short_id(self):
@@ -301,26 +292,25 def repo2db_mapper(initial_repo_list, re
     """
     maps all found repositories into db
     """
-    from rhodecode.model.repo_model import RepoModel
 
-    sa = meta.Session
+    sa = meta.Session()
+    rm = RepoModel()
     user = sa.query(User).filter(User.admin == True).first()
 
-    rm = RepoModel()
-
     for name, repo in initial_repo_list.items():
-        if not sa.query(Repository).filter(Repository.repo_name == name).scalar():
+        if not rm.get_by_repo_name(name, cache=False):
             log.info('repository %s not found creating default', name)
 
             form_data = {
                          'repo_name':name,
-                         'description':repo.description if repo.description != 'unknown' else \
-                                        'auto description for %s' % name,
+                         'repo_type':repo.alias,
+                         'description':repo.description \
+                            if repo.description != 'unknown' else \
+                                        '%s repository' % name,
                          'private':False
                          }
             rm.create(form_data, user, just_db=True)
 
-
     if remove_obsolete:
         #remove from database those repositories that are not in the filesystem
         for repo in sa.query(Repository).all():
@@ -328,11 +318,6 def repo2db_mapper(initial_repo_list, re
                 sa.delete(repo)
                 sa.commit()
 
-
-    meta.Session.remove()
-
-from UserDict import DictMixin
-
 class OrderedDict(dict, DictMixin):
 
     def __init__(self, *args, **kwds):
@@ -433,8 +418,51 class OrderedDict(dict, DictMixin):
         return not self == other
 
 
+#set cache regions for beaker so celery can utilise it
+def add_cache(settings):
+    cache_settings = {'regions':None}
+    for key in settings.keys():
+        for prefix in ['beaker.cache.', 'cache.']:
+            if key.startswith(prefix):
+                name = key.split(prefix)[1].strip()
+                cache_settings[name] = settings[key].strip()
+    if cache_settings['regions']:
+        for region in cache_settings['regions'].split(','):
+            region = region.strip()
+            region_settings = {}
+            for key, value in cache_settings.items():
+                if key.startswith(region):
+                    region_settings[key.split('.')[1]] = value
+            region_settings['expire'] = int(region_settings.get('expire',
+                                                                60))
+            region_settings.setdefault('lock_dir',
+                                       cache_settings.get('lock_dir'))
+            if 'type' not in region_settings:
+                region_settings['type'] = cache_settings.get('type',
+                                                             'memory')
+            beaker.cache.cache_regions[region] = region_settings
+
+def get_current_revision():
+    """
+    Returns tuple of (number, id) from repository containing this package
+    or None if repository could not be found.
+    """
+    try:
+        from vcs import get_repo
+        from vcs.utils.helpers import get_scm
+        from vcs.exceptions import RepositoryError, VCSError
+        repopath = os.path.join(os.path.dirname(__file__), '..', '..')
+        scm = get_scm(repopath)[0]
+        repo = get_repo(path=repopath, alias=scm)
+        tip = repo.get_changeset()
+        return (tip.revision, tip.short_id)
+    except (ImportError, RepositoryError, VCSError), err:
+        logging.debug("Cannot retrieve rhodecode's revision. Original error "
+                      "was: %s" % err)
+        return None
+
 #===============================================================================
-# TEST FUNCTIONS
+# TEST FUNCTIONS AND CREATORS
 #===============================================================================
 def create_test_index(repo_location, full_index):
     """Makes default test index
@@ -443,15 +471,16 def create_test_index(repo_location, ful
     """
     from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
     from rhodecode.lib.pidlock import DaemonLock, LockHeld
-    from rhodecode.lib.indexers import IDX_LOCATION
     import shutil
 
-    if os.path.exists(IDX_LOCATION):
-        shutil.rmtree(IDX_LOCATION)
+    index_location = os.path.join(repo_location, 'index')
+    if os.path.exists(index_location):
+        shutil.rmtree(index_location)
 
     try:
         l = DaemonLock()
-        WhooshIndexingDaemon(repo_location=repo_location)\
+        WhooshIndexingDaemon(index_location=index_location,
+                             repo_location=repo_location)\
             .run(full_index=full_index)
         l.release()
     except LockHeld:
@@ -462,10 +491,11 def create_test_env(repos_test_path, con
     install test repository into tmp dir
     """
     from rhodecode.lib.db_manage import DbManage
+    from rhodecode.tests import HG_REPO, GIT_REPO, NEW_HG_REPO, NEW_GIT_REPO, \
+        HG_FORK, GIT_FORK, TESTS_TMP_PATH
     import tarfile
     import shutil
     from os.path import dirname as dn, join as jn, abspath
-    from rhodecode.tests import REPO_PATH, NEW_REPO_PATH, FORK_REPO_PATH
 
     log = logging.getLogger('TestEnvCreator')
     # create logger
@@ -485,10 +515,10 def create_test_env(repos_test_path, con
     log.addHandler(ch)
 
     #PART ONE create db
-    dbname = config['sqlalchemy.db1.url'].split('/')[-1]
-    log.debug('making test db %s', dbname)
+    dbconf = config['sqlalchemy.db1.url']
+    log.debug('making test db %s', dbconf)
 
-    dbmanage = DbManage(log_sql=True, dbname=dbname, root=config['here'],
+    dbmanage = DbManage(log_sql=True, dbconf=dbconf, root=config['here'],
                         tests=True)
     dbmanage.create_tables(override=True)
     dbmanage.config_prompt(repos_test_path)
@@ -498,18 +528,87 def create_test_env(repos_test_path, con
     dbmanage.populate_default_permissions()
 
     #PART TWO make test repo
-    log.debug('making test vcs repo')
-    if os.path.isdir(REPO_PATH):
-        log.debug('REMOVING %s', REPO_PATH)
-        shutil.rmtree(REPO_PATH)
-    if os.path.isdir(NEW_REPO_PATH):
-        log.debug('REMOVING %s', NEW_REPO_PATH)
-        shutil.rmtree(NEW_REPO_PATH)
-    if os.path.isdir(FORK_REPO_PATH):
-        log.debug('REMOVING %s', FORK_REPO_PATH)
-        shutil.rmtree(FORK_REPO_PATH)
+    log.debug('making test vcs repositories')
+
+    #remove old one from previos tests
+    for r in [HG_REPO, GIT_REPO, NEW_HG_REPO, NEW_GIT_REPO, HG_FORK, GIT_FORK]:
+
+        if os.path.isdir(jn(TESTS_TMP_PATH, r)):
+            log.debug('removing %s', r)
+            shutil.rmtree(jn(TESTS_TMP_PATH, r))
+
+    #CREATE DEFAULT HG REPOSITORY
+    cur_dir = dn(dn(abspath(__file__)))
+    tar = tarfile.open(jn(cur_dir, 'tests', "vcs_test_hg.tar.gz"))
+    tar.extractall(jn(TESTS_TMP_PATH, HG_REPO))
+    tar.close()
+
+
+#==============================================================================
+# PASTER COMMANDS
+#==============================================================================
+
+class BasePasterCommand(Command):
+    """
+    Abstract Base Class for paster commands.
+
+    The celery commands are somewhat aggressive about loading
+    celery.conf, and since our module sets the `CELERY_LOADER`
+    environment variable to our loader, we have to bootstrap a bit and
+    make sure we've had a chance to load the pylons config off of the
+    command line, otherwise everything fails.
+    """
+    min_args = 1
+    min_args_error = "Please provide a paster config file as an argument."
+    takes_config_file = 1
+    requires_config_file = True
+
+    def notify_msg(self, msg, log=False):
+        """Make a notification to user, additionally if logger is passed
+        it logs this action using given logger
+        
+        :param msg: message that will be printed to user
+        :param log: logging instance, to use to additionally log this message
 
-    cur_dir = dn(dn(abspath(__file__)))
-    tar = tarfile.open(jn(cur_dir, 'tests', "vcs_test.tar.gz"))
-    tar.extractall('/tmp')
-    tar.close()
+        """
+        print msg
+        if log and isinstance(log, logging):
+            log(msg)
+
+
+    def run(self, args):
+        """
+        Overrides Command.run
+        
+        Checks for a config file argument and loads it.
+        """
+        if len(args) < self.min_args:
+            raise BadCommand(
+                self.min_args_error % {'min_args': self.min_args,
+                                       'actual_args': len(args)})
+
+        # Decrement because we're going to lob off the first argument.
+        # @@ This is hacky
+        self.min_args -= 1
+        self.bootstrap_config(args[0])
+        self.update_parser()
+        return super(BasePasterCommand, self).run(args[1:])
+
+    def update_parser(self):
+        """
+        Abstract method.  Allows for the class's parser to be updated
+        before the superclass's `run` method is called.  Necessary to
+        allow options/arguments to be passed through to the underlying
+        celery command.
+        """
+        raise NotImplementedError("Abstract Method.")
+
+    def bootstrap_config(self, conf):
+        """
+        Loads the pylons configuration.
+        """
+        from pylons import config as pylonsconfig
+
+        path_to_ini_file = os.path.realpath(conf)
+        conf = paste.deploy.appconfig('config:' + path_to_ini_file)
+        pylonsconfig.init_app(conf.global_conf, conf.local_conf)

rhodecode/model/__init__.py

@@ -1,23 +1,71
-"""The application's model objects"""
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.model.__init__
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+    
+    The application's model objects
+    
+    :created_on: Nov 25, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+    
+    
+    :example:
+    
+        .. code-block:: python
+    
+           from paste.deploy import appconfig
+           from pylons import config
+           from sqlalchemy import engine_from_config
+           from rhodecode.config.environment import load_environment
+           
+           conf = appconfig('config:development.ini', relative_to = './../../')
+           load_environment(conf.global_conf, conf.local_conf)
+           
+           engine = engine_from_config(config, 'sqlalchemy.')
+           init_model(engine)
+           # RUN YOUR CODE HERE
+    
+"""
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your opinion) any later version of the license.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+
 import logging
 from rhodecode.model import meta
 log = logging.getLogger(__name__)
 
 def init_model(engine):
-    """Call me before using any of the tables or classes in the model"""
-    log.info("INITIALIZING DB MODELS")
+    """Initializes db session, bind the engine with the metadata,
+    Call this before using any of the tables or classes in the model, preferably
+    once in application start
+    
+    :param engine: engine to bind to
+    """
+    log.info("initializing db models for %s", engine)
     meta.Base.metadata.bind = engine
-    #meta.Base2.metadata.bind = engine2
 
-#THIS IS A TEST FOR EXECUTING SCRIPT AND LOAD PYLONS APPLICATION GLOBALS
-#from paste.deploy import appconfig
-#from pylons import config
-#from sqlalchemy import engine_from_config
-#from rhodecode.config.environment import load_environment
-#
-#conf = appconfig('config:development.ini', relative_to = './../../')
-#load_environment(conf.global_conf, conf.local_conf)
-#
-#engine = engine_from_config(config, 'sqlalchemy.')
-#init_model(engine)
-# DO SOMETHING
+class BaseModel(object):
+    """Base Model for all RhodeCode models, it adds sql alchemy session
+    into instance of model
+    
+    :param sa: If passed it reuses this session instead of creating a new one
+    """
+
+    def __init__(self, sa=None):
+        if sa is not None:
+            self.sa = sa
+        else:
+            self.sa = meta.Session()

rhodecode/model/caching_query.py

@@ -110,7 +110,7 def query_callable(manager):
 
 def get_cache_region(name, region):
     if region not in beaker.cache.cache_regions:
-        raise BeakerException('Cache region not configured: %s'
+        raise BeakerException('Cache region `%s` not configured '
             'Check if proper cache settings are in the .ini files' % region)
     kw = beaker.cache.cache_regions[region]
     return beaker.cache.Cache._get_cache(name, kw)

rhodecode/model/db.py

@@ -1,46 +1,119
-from rhodecode.model.meta import Base
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.model.db
+    ~~~~~~~~~~~~~~~~~~
+    
+    Database Models for RhodeCode
+    
+    :created_on: Apr 08, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your opinion) any later version of the license.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+import logging
+import datetime
+
 from sqlalchemy import *
-from sqlalchemy.orm import relation, backref
+from sqlalchemy.exc import DatabaseError
+from sqlalchemy.orm import relation, backref, class_mapper
 from sqlalchemy.orm.session import Session
-from vcs.utils.lazy import LazyProperty
-import logging
+
+from rhodecode.model.meta import Base
 
 log = logging.getLogger(__name__)
 
-class RhodeCodeSettings(Base):
+class BaseModel(object):
+
+    @classmethod
+    def _get_keys(cls):
+        """return column names for this model """
+        return class_mapper(cls).c.keys()
+
+    def get_dict(self):
+        """return dict with keys and values corresponding 
+        to this model data """
+
+        d = {}
+        for k in self._get_keys():
+            d[k] = getattr(self, k)
+        return d
+
+    def get_appstruct(self):
+        """return list with keys and values tupples corresponding 
+        to this model data """
+
+        l = []
+        for k in self._get_keys():
+            l.append((k, getattr(self, k),))
+        return l
+
+    def populate_obj(self, populate_dict):
+        """populate model with data from given populate_dict"""
+
+        for k in self._get_keys():
+            if k in populate_dict:
+                setattr(self, k, populate_dict[k])
+
+class RhodeCodeSettings(Base, BaseModel):
     __tablename__ = 'rhodecode_settings'
     __table_args__ = (UniqueConstraint('app_settings_name'), {'useexisting':True})
-    app_settings_id = Column("app_settings_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
-    app_settings_name = Column("app_settings_name", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
-    app_settings_value = Column("app_settings_value", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    app_settings_id = Column("app_settings_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    app_settings_name = Column("app_settings_name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    app_settings_value = Column("app_settings_value", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
-class RhodeCodeUi(Base):
+    def __init__(self, k, v):
+        self.app_settings_name = k
+        self.app_settings_value = v
+
+    def __repr__(self):
+        return "<RhodeCodeSetting('%s:%s')>" % (self.app_settings_name,
+                                                self.app_settings_value)
+
+class RhodeCodeUi(Base, BaseModel):
     __tablename__ = 'rhodecode_ui'
     __table_args__ = {'useexisting':True}
-    ui_id = Column("ui_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
-    ui_section = Column("ui_section", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
-    ui_key = Column("ui_key", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
-    ui_value = Column("ui_value", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
-    ui_active = Column("ui_active", BOOLEAN(), nullable=True, unique=None, default=True)
+    ui_id = Column("ui_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    ui_section = Column("ui_section", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    ui_key = Column("ui_key", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    ui_value = Column("ui_value", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    ui_active = Column("ui_active", Boolean(), nullable=True, unique=None, default=True)
 
 
-class User(Base):
+class User(Base, BaseModel):
     __tablename__ = 'users'
     __table_args__ = (UniqueConstraint('username'), UniqueConstraint('email'), {'useexisting':True})
-    user_id = Column("user_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
-    username = Column("username", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
-    password = Column("password", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
-    active = Column("active", BOOLEAN(), nullable=True, unique=None, default=None)
-    admin = Column("admin", BOOLEAN(), nullable=True, unique=None, default=False)
-    name = Column("name", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
-    lastname = Column("lastname", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
-    email = Column("email", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
-    last_login = Column("last_login", DATETIME(timezone=False), nullable=True, unique=None, default=None)
+    user_id = Column("user_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    username = Column("username", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    password = Column("password", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    active = Column("active", Boolean(), nullable=True, unique=None, default=None)
+    admin = Column("admin", Boolean(), nullable=True, unique=None, default=False)
+    name = Column("name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    lastname = Column("lastname", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    email = Column("email", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    last_login = Column("last_login", DateTime(timezone=False), nullable=True, unique=None, default=None)
+    is_ldap = Column("is_ldap", Boolean(), nullable=False, unique=None, default=False)
 
-    user_log = relation('UserLog')
-    user_perms = relation('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id")
+    user_log = relation('UserLog', cascade='all')
+    user_perms = relation('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id", cascade='all')
 
-    @LazyProperty
+    repositories = relation('Repository')
+    user_followers = relation('UserFollowing', primaryjoin='UserFollowing.follows_user_id==User.user_id', cascade='all')
+
+    @property
     def full_contact(self):
         return '%s %s <%s>' % (self.name, self.lastname, self.email)
 
@@ -49,7 +122,6 class User(Base):
 
     def update_lastlogin(self):
         """Update user lastlogin"""
-        import datetime
 
         try:
             session = Session.object_session(self)
@@ -57,85 +129,129 class User(Base):
             session.add(self)
             session.commit()
             log.debug('updated user %s lastlogin', self.username)
-        except Exception:
+        except (DatabaseError,):
             session.rollback()
 
 
-class UserLog(Base):
+class UserLog(Base, BaseModel):
     __tablename__ = 'user_logs'
     __table_args__ = {'useexisting':True}
-    user_log_id = Column("user_log_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
-    user_id = Column("user_id", INTEGER(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
-    repository_id = Column("repository_id", INTEGER(length=None, convert_unicode=False, assert_unicode=None), ForeignKey(u'repositories.repo_id'), nullable=False, unique=None, default=None)
-    repository_name = Column("repository_name", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
-    user_ip = Column("user_ip", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
-    action = Column("action", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
-    action_date = Column("action_date", DATETIME(timezone=False), nullable=True, unique=None, default=None)
-    revision = Column('revision', TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    user_log_id = Column("user_log_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
+    repository_id = Column("repository_id", Integer(length=None, convert_unicode=False, assert_unicode=None), ForeignKey(u'repositories.repo_id'), nullable=False, unique=None, default=None)
+    repository_name = Column("repository_name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    user_ip = Column("user_ip", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    action = Column("action", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    action_date = Column("action_date", DateTime(timezone=False), nullable=True, unique=None, default=None)
 
     user = relation('User')
     repository = relation('Repository')
 
-class Repository(Base):
+class Repository(Base, BaseModel):
     __tablename__ = 'repositories'
     __table_args__ = (UniqueConstraint('repo_name'), {'useexisting':True},)
-    repo_id = Column("repo_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
-    repo_name = Column("repo_name", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
-    user_id = Column("user_id", INTEGER(), ForeignKey(u'users.user_id'), nullable=False, unique=False, default=None)
-    private = Column("private", BOOLEAN(), nullable=True, unique=None, default=None)
-    description = Column("description", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
-    fork_id = Column("fork_id", INTEGER(), ForeignKey(u'repositories.repo_id'), nullable=True, unique=False, default=None)
+    repo_id = Column("repo_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    repo_name = Column("repo_name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
+    repo_type = Column("repo_type", String(length=None, convert_unicode=False, assert_unicode=None), nullable=False, unique=False, default='hg')
+    user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=False, default=None)
+    private = Column("private", Boolean(), nullable=True, unique=None, default=None)
+    enable_statistics = Column("statistics", Boolean(), nullable=True, unique=None, default=True)
+    description = Column("description", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    fork_id = Column("fork_id", Integer(), ForeignKey(u'repositories.repo_id'), nullable=True, unique=False, default=None)
 
     user = relation('User')
     fork = relation('Repository', remote_side=repo_id)
     repo_to_perm = relation('RepoToPerm', cascade='all')
-    stats = relation('Statistics', cascade='all')
+    stats = relation('Statistics', cascade='all', uselist=False)
+
+    repo_followers = relation('UserFollowing', primaryjoin='UserFollowing.follows_repo_id==Repository.repo_id', cascade='all')
 
 
     def __repr__(self):
-        return "<Repository('id:%s:%s')>" % (self.repo_id, self.repo_name)
+        return "<Repository('%s:%s')>" % (self.repo_id, self.repo_name)
 
-class Permission(Base):
+class Permission(Base, BaseModel):
     __tablename__ = 'permissions'
     __table_args__ = {'useexisting':True}
-    permission_id = Column("permission_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
-    permission_name = Column("permission_name", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
-    permission_longname = Column("permission_longname", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    permission_id = Column("permission_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    permission_name = Column("permission_name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    permission_longname = Column("permission_longname", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
     def __repr__(self):
         return "<Permission('%s:%s')>" % (self.permission_id, self.permission_name)
 
-class RepoToPerm(Base):
+class RepoToPerm(Base, BaseModel):
     __tablename__ = 'repo_to_perm'
     __table_args__ = (UniqueConstraint('user_id', 'repository_id'), {'useexisting':True})
-    repo_to_perm_id = Column("repo_to_perm_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
-    user_id = Column("user_id", INTEGER(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
-    permission_id = Column("permission_id", INTEGER(), ForeignKey(u'permissions.permission_id'), nullable=False, unique=None, default=None)
-    repository_id = Column("repository_id", INTEGER(), ForeignKey(u'repositories.repo_id'), nullable=False, unique=None, default=None)
+    repo_to_perm_id = Column("repo_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
+    permission_id = Column("permission_id", Integer(), ForeignKey(u'permissions.permission_id'), nullable=False, unique=None, default=None)
+    repository_id = Column("repository_id", Integer(), ForeignKey(u'repositories.repo_id'), nullable=False, unique=None, default=None)
 
     user = relation('User')
     permission = relation('Permission')
     repository = relation('Repository')
 
-class UserToPerm(Base):
+class UserToPerm(Base, BaseModel):
     __tablename__ = 'user_to_perm'
     __table_args__ = (UniqueConstraint('user_id', 'permission_id'), {'useexisting':True})
-    user_to_perm_id = Column("user_to_perm_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
-    user_id = Column("user_id", INTEGER(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
-    permission_id = Column("permission_id", INTEGER(), ForeignKey(u'permissions.permission_id'), nullable=False, unique=None, default=None)
+    user_to_perm_id = Column("user_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
+    permission_id = Column("permission_id", Integer(), ForeignKey(u'permissions.permission_id'), nullable=False, unique=None, default=None)
 
     user = relation('User')
     permission = relation('Permission')
 
-class Statistics(Base):
+class Statistics(Base, BaseModel):
     __tablename__ = 'statistics'
     __table_args__ = (UniqueConstraint('repository_id'), {'useexisting':True})
-    stat_id = Column("stat_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
-    repository_id = Column("repository_id", INTEGER(), ForeignKey(u'repositories.repo_id'), nullable=False, unique=True, default=None)
-    stat_on_revision = Column("stat_on_revision", INTEGER(), nullable=False)
-    commit_activity = Column("commit_activity", BLOB(), nullable=False)#JSON data
-    commit_activity_combined = Column("commit_activity_combined", BLOB(), nullable=False)#JSON data
-    languages = Column("languages", BLOB(), nullable=False)#JSON data
+    stat_id = Column("stat_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    repository_id = Column("repository_id", Integer(), ForeignKey(u'repositories.repo_id'), nullable=False, unique=True, default=None)
+    stat_on_revision = Column("stat_on_revision", Integer(), nullable=False)
+    commit_activity = Column("commit_activity", LargeBinary(), nullable=False)#JSON data
+    commit_activity_combined = Column("commit_activity_combined", LargeBinary(), nullable=False)#JSON data
+    languages = Column("languages", LargeBinary(), nullable=False)#JSON data
 
     repository = relation('Repository', single_parent=True)
 
+class UserFollowing(Base, BaseModel):
+    __tablename__ = 'user_followings'
+    __table_args__ = (UniqueConstraint('user_id', 'follows_repository_id'),
+                      UniqueConstraint('user_id', 'follows_user_id')
+                      , {'useexisting':True})
+
+    user_following_id = Column("user_following_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
+    follows_repo_id = Column("follows_repository_id", Integer(), ForeignKey(u'repositories.repo_id'), nullable=True, unique=None, default=None)
+    follows_user_id = Column("follows_user_id", Integer(), ForeignKey(u'users.user_id'), nullable=True, unique=None, default=None)
+
+    user = relation('User', primaryjoin='User.user_id==UserFollowing.user_id')
+
+    follows_user = relation('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')
+    follows_repository = relation('Repository')
+
+
+class CacheInvalidation(Base, BaseModel):
+    __tablename__ = 'cache_invalidation'
+    __table_args__ = (UniqueConstraint('cache_key'), {'useexisting':True})
+    cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    cache_key = Column("cache_key", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    cache_args = Column("cache_args", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
+    cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False)
+
+
+    def __init__(self, cache_key, cache_args=''):
+        self.cache_key = cache_key
+        self.cache_args = cache_args
+        self.cache_active = False
+
+    def __repr__(self):
+        return "<CacheInvalidation('%s:%s')>" % (self.cache_id, self.cache_key)
+
+class DbMigrateVersion(Base, BaseModel):
+    __tablename__ = 'db_migrate_version'
+    __table_args__ = {'useexisting':True}
+    repository_id = Column('repository_id', String(250), primary_key=True)
+    repository_path = Column('repository_path', Text)
+    version = Column('version', Integer)
+

rhodecode/model/forms.py

@@ -19,25 +19,30 list=[1,2,3,4,5]
 for SELECT use formencode.All(OneOf(list), Int())
     
 """
+import os
+import re
+import logging
+
+import formencode
 from formencode import All
 from formencode.validators import UnicodeString, OneOf, Int, Number, Regex, \
     Email, Bool, StringBoolean
-from pylons import session
+
 from pylons.i18n.translation import _
-from rhodecode.lib.auth import check_password, get_crypt_password
+
+import rhodecode.lib.helpers as h
+from rhodecode.lib.auth import authenticate, get_crypt_password
+from rhodecode.lib.exceptions import LdapImportError
 from rhodecode.model import meta
-from rhodecode.model.user_model import UserModel
-from rhodecode.model.db import User, Repository
-from sqlalchemy.exc import OperationalError
-from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
+from rhodecode.model.user import UserModel
+from rhodecode.model.repo import RepoModel
+from rhodecode.model.db import User
+from rhodecode import BACKENDS
+
 from webhelpers.pylonslib.secure_form import authentication_token
-import formencode
-import logging
-import os
-import rhodecode.lib.helpers as h
+
 log = logging.getLogger(__name__)
 
-
 #this is needed to translate the messages using _() in validators
 class State_obj(object):
     _ = staticmethod(_)
@@ -60,31 +65,74 def ValidUsername(edit, old_data):
         def validate_python(self, value, state):
             if value in ['default', 'new_user']:
                 raise formencode.Invalid(_('Invalid username'), value, state)
-            #check if user is uniq
-            sa = meta.Session
+            #check if user is unique
             old_un = None
             if edit:
-                old_un = sa.query(User).get(old_data.get('user_id')).username
+                old_un = UserModel().get(old_data.get('user_id')).username
                 
             if old_un != value or not edit:    
-                if sa.query(User).filter(User.username == value).scalar():
+                if UserModel().get_by_username(value, cache=False,
+                                               case_insensitive=True):
                     raise formencode.Invalid(_('This username already exists') ,
                                              value, state)
-            meta.Session.remove()
+
+
+            if re.match(r'^[a-zA-Z0-9]{1}[a-zA-Z0-9\-\_]+$', value) is None:
+                raise formencode.Invalid(_('Username may only contain '
+                                           'alphanumeric characters underscores '
+                                           'or dashes and must begin with '
+                                           'alphanumeric character'),
+                                      value, state)
+
+
                             
     return _ValidUsername   
     
 class ValidPassword(formencode.validators.FancyValidator):
     
     def to_python(self, value, state):
+
         if value:
-            return get_crypt_password(value)
+
+            if value.get('password'):
+                try:
+                    value['password'] = get_crypt_password(value['password'])
+                except UnicodeEncodeError:
+                    e_dict = {'password':_('Invalid characters in password')}
+                    raise formencode.Invalid('', value, state, error_dict=e_dict)
+
+            if value.get('password_confirmation'):
+                try:
+                    value['password_confirmation'] = \
+                        get_crypt_password(value['password_confirmation'])
+                except UnicodeEncodeError:
+                    e_dict = {'password_confirmation':_('Invalid characters in password')}
+                    raise formencode.Invalid('', value, state, error_dict=e_dict)
+
+            if value.get('new_password'):
+                try:
+                    value['new_password'] = \
+                        get_crypt_password(value['new_password'])
+                except UnicodeEncodeError:
+                    e_dict = {'new_password':_('Invalid characters in password')}
+                    raise formencode.Invalid('', value, state, error_dict=e_dict)
+
+            return value
+
+class ValidPasswordsMatch(formencode.validators.FancyValidator):
+
+    def validate_python(self, value, state):
+
+        if value['password'] != value['password_confirmation']:
+            e_dict = {'password_confirmation':
+                   _('Password do not match')}
+            raise formencode.Invalid('', value, state, error_dict=e_dict)
         
 class ValidAuth(formencode.validators.FancyValidator):
     messages = {
             'invalid_password':_('invalid password'),
             'invalid_login':_('invalid user name'),
-            'disabled_account':_('Your acccount is disabled')
+            'disabled_account':_('Your account is disabled')
             
             }
     #error mapping
@@ -95,33 +143,29 class ValidAuth(formencode.validators.Fa
     def validate_python(self, value, state):
         password = value['password']
         username = value['username']
-        user = UserModel().get_user_by_name(username)
-        if user is None:
-            raise formencode.Invalid(self.message('invalid_password',
-                                     state=State_obj), value, state,
-                                     error_dict=self.e_dict)            
-        if user:
-            if user.active:
-                if user.username == username and check_password(password,
-                                                                user.password):
+        user = UserModel().get_by_username(username)
+
+        if authenticate(username, password):
                     return value
                 else:
+            if user and user.active is False:
+                log.warning('user %s is disabled', username)
+                raise formencode.Invalid(self.message('disabled_account',
+                                         state=State_obj),
+                                         value, state,
+                                         error_dict=self.e_dict_disable)
+            else:
                     log.warning('user %s not authenticated', username)
                     raise formencode.Invalid(self.message('invalid_password',
                                              state=State_obj), value, state,
                                              error_dict=self.e_dict)
-            else:
-                log.warning('user %s is disabled', username)
-                raise formencode.Invalid(self.message('disabled_account',
-                                         state=State_obj),
-                                         value, state,
-                                         error_dict=self.e_dict_disable)
                    
 class ValidRepoUser(formencode.validators.FancyValidator):
             
     def to_python(self, value, state):
+        sa = meta.Session()
         try:
-            self.user_db = meta.Session.query(User)\
+            self.user_db = sa.query(User)\
                 .filter(User.active == True)\
                 .filter(User.username == value).one()
         except Exception:
@@ -141,16 +185,24 def ValidRepoName(edit, old_data):
                 raise formencode.Invalid(_('This repository name is disallowed'),
                                          value, state)
             if old_data.get('repo_name') != value or not edit:    
-                sa = meta.Session
-                if sa.query(Repository).filter(Repository.repo_name == slug).scalar():
+                if RepoModel().get_by_repo_name(slug, cache=False):
                     raise formencode.Invalid(_('This repository already exists') ,
                                              value, state)
-                meta.Session.remove()
             return slug 
         
         
     return _ValidRepoName
 
+def ValidForkType(old_data):
+    class _ValidForkType(formencode.validators.FancyValidator):
+
+        def to_python(self, value, state):
+            if old_data['repo_type'] != value:
+                raise formencode.Invalid(_('Fork have to be the same type as original'),
+                                         value, state)
+            return value
+    return _ValidForkType
+
 class ValidPerms(formencode.validators.FancyValidator):
     messages = {'perm_new_user_name':_('This username is not valid')}
     
@@ -184,7 +236,8 class ValidPerms(formencode.validators.F
             except Exception:
                 msg = self.message('perm_new_user_name',
                                      state=State_obj)
-                raise formencode.Invalid(msg, value, state, error_dict={'perm_new_user_name':msg})            
+                raise formencode.Invalid(msg, value, state,
+                                         error_dict={'perm_new_user_name':msg})
         return value
     
 class ValidSettings(formencode.validators.FancyValidator):
@@ -198,26 +251,23 class ValidSettings(formencode.validator
     
 class ValidPath(formencode.validators.FancyValidator):
     def to_python(self, value, state):
-        isdir = os.path.isdir(value.replace('*', ''))
-        if (value.endswith('/*') or value.endswith('/**')) and isdir:
-            return value
-        elif not isdir:
+
+        if not os.path.isdir(value):
             msg = _('This is not a valid path') 
-        else:
-            msg = _('You need to specify * or ** at the end of path (ie. /tmp/*)')
-        
         raise formencode.Invalid(msg, value, state,
                                      error_dict={'paths_root_path':msg})            
+        return value
 
 def UniqSystemEmail(old_data):
     class _UniqSystemEmail(formencode.validators.FancyValidator):
         def to_python(self, value, state):
+            value = value.lower()
             if old_data.get('email') != value:
-                sa = meta.Session
+                sa = meta.Session()
                 try:
                     user = sa.query(User).filter(User.email == value).scalar()
                     if user:
-                        raise formencode.Invalid(_("That e-mail address is already taken") ,
+                        raise formencode.Invalid(_("This e-mail address is already taken") ,
                                                  value, state)
                 finally:
                     meta.Session.remove()
@@ -228,17 +278,48 def UniqSystemEmail(old_data):
     
 class ValidSystemEmail(formencode.validators.FancyValidator):
     def to_python(self, value, state):
+        value = value.lower()
         sa = meta.Session
         try:
             user = sa.query(User).filter(User.email == value).scalar()
             if  user is None:
-                raise formencode.Invalid(_("That e-mail address doesn't exist.") ,
+                raise formencode.Invalid(_("This e-mail address doesn't exist.") ,
                                          value, state)
         finally:
             meta.Session.remove()
             
         return value     
 
+class LdapLibValidator(formencode.validators.FancyValidator):
+
+    def to_python(self, value, state):
+
+        try:
+            import ldap
+        except ImportError:
+            raise LdapImportError
+        return value
+
+class BaseDnValidator(formencode.validators.FancyValidator):
+
+    def to_python(self, value, state):
+
+        try:
+            value % {'user':'valid'}
+
+            if value.find('%(user)s') == -1:
+                raise formencode.Invalid(_("You need to specify %(user)s in "
+                                           "template for example uid=%(user)s "
+                                           ",dc=company...") ,
+                                         value, state)
+
+        except KeyError:
+            raise formencode.Invalid(_("Wrong template used, only %(user)s "
+                                       "is an valid entry") ,
+                                         value, state)
+
+        return value
+
 #===============================================================================
 # FORMS        
 #===============================================================================
@@ -271,20 +352,38 def UserForm(edit=False, old_data={}):
     class _UserForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
-        username = All(UnicodeString(strip=True, min=1, not_empty=True), ValidUsername(edit, old_data))
+        username = All(UnicodeString(strip=True, min=1, not_empty=True),
+                       ValidUsername(edit, old_data))
         if edit:
-            new_password = All(UnicodeString(strip=True, min=6, not_empty=False), ValidPassword)
+            new_password = All(UnicodeString(strip=True, min=6, not_empty=False))
             admin = StringBoolean(if_missing=False)
         else:
-            password = All(UnicodeString(strip=True, min=6, not_empty=True), ValidPassword)
+            password = All(UnicodeString(strip=True, min=6, not_empty=True))
         active = StringBoolean(if_missing=False)
         name = UnicodeString(strip=True, min=1, not_empty=True)
         lastname = UnicodeString(strip=True, min=1, not_empty=True)
         email = All(Email(not_empty=True), UniqSystemEmail(old_data))
         
+        chained_validators = [ValidPassword]
+
     return _UserForm
 
-RegisterForm = UserForm
+def RegisterForm(edit=False, old_data={}):
+    class _RegisterForm(formencode.Schema):
+        allow_extra_fields = True
+        filter_extra_fields = True
+        username = All(ValidUsername(edit, old_data),
+                       UnicodeString(strip=True, min=1, not_empty=True))
+        password = All(UnicodeString(strip=True, min=6, not_empty=True))
+        password_confirmation = All(UnicodeString(strip=True, min=6, not_empty=True))
+        active = StringBoolean(if_missing=False)
+        name = UnicodeString(strip=True, min=1, not_empty=True)
+        lastname = UnicodeString(strip=True, min=1, not_empty=True)
+        email = All(Email(not_empty=True), UniqSystemEmail(old_data))
+
+        chained_validators = [ValidPasswordsMatch, ValidPassword]
+
+    return _RegisterForm
 
 def PasswordResetForm():
     class _PasswordResetForm(formencode.Schema):
@@ -293,35 +392,39 def PasswordResetForm():
         email = All(ValidSystemEmail(), Email(not_empty=True))             
     return _PasswordResetForm
 
-def RepoForm(edit=False, old_data={}):
+def RepoForm(edit=False, old_data={}, supported_backends=BACKENDS.keys()):
     class _RepoForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
-        repo_name = All(UnicodeString(strip=True, min=1, not_empty=True), ValidRepoName(edit, old_data))
+        repo_name = All(UnicodeString(strip=True, min=1, not_empty=True),
+                        ValidRepoName(edit, old_data))
         description = UnicodeString(strip=True, min=1, not_empty=True)
         private = StringBoolean(if_missing=False)
-        
+        enable_statistics = StringBoolean(if_missing=False)
+        repo_type = OneOf(supported_backends)
         if edit:
             user = All(Int(not_empty=True), ValidRepoUser)
         
         chained_validators = [ValidPerms]
     return _RepoForm
 
-def RepoForkForm(edit=False, old_data={}):
+def RepoForkForm(edit=False, old_data={}, supported_backends=BACKENDS.keys()):
     class _RepoForkForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
-        fork_name = All(UnicodeString(strip=True, min=1, not_empty=True), ValidRepoName(edit, old_data))
+        fork_name = All(UnicodeString(strip=True, min=1, not_empty=True),
+                        ValidRepoName(edit, old_data))
         description = UnicodeString(strip=True, min=1, not_empty=True)
         private = StringBoolean(if_missing=False)
-        
+        repo_type = All(ValidForkType(old_data), OneOf(supported_backends))
     return _RepoForkForm
 
 def RepoSettingsForm(edit=False, old_data={}):
     class _RepoForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
-        repo_name = All(UnicodeString(strip=True, min=1, not_empty=True), ValidRepoName(edit, old_data))
+        repo_name = All(UnicodeString(strip=True, min=1, not_empty=True),
+                        ValidRepoName(edit, old_data))
         description = UnicodeString(strip=True, min=1, not_empty=True)
         private = StringBoolean(if_missing=False)
         
@@ -346,6 +449,8 def ApplicationUiSettingsForm():
         paths_root_path = All(ValidPath(), UnicodeString(strip=True, min=1, not_empty=True))
         hooks_changegroup_update = OneOf(['True', 'False'], if_missing=False)
         hooks_changegroup_repo_size = OneOf(['True', 'False'], if_missing=False)
+        hooks_pretxnchangegroup_push_logger = OneOf(['True', 'False'], if_missing=False)
+        hooks_preoutgoing_pull_logger = OneOf(['True', 'False'], if_missing=False)
         
     return _ApplicationUiSettingsForm
 
@@ -353,9 +458,26 def DefaultPermissionsForm(perms_choices
     class _DefaultPermissionsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
-        overwrite_default = OneOf(['true', 'false'], if_missing='false')
+        overwrite_default = StringBoolean(if_missing=False)
+        anonymous = OneOf(['True', 'False'], if_missing=False)
         default_perm = OneOf(perms_choices)
         default_register = OneOf(register_choices)
         default_create = OneOf(create_choices)
         
     return _DefaultPermissionsForm
+
+
+def LdapSettingsForm():
+    class _LdapSettingsForm(formencode.Schema):
+        allow_extra_fields = True
+        filter_extra_fields = True
+        pre_validators = [LdapLibValidator]
+        ldap_active = StringBoolean(if_missing=False)
+        ldap_host = UnicodeString(strip=True,)
+        ldap_port = Number(strip=True,)
+        ldap_ldaps = StringBoolean(if_missing=False)
+        ldap_dn_user = UnicodeString(strip=True,)
+        ldap_dn_pass = UnicodeString(strip=True,)
+        ldap_base_dn = All(BaseDnValidator, UnicodeString(strip=True,))
+
+    return _LdapSettingsForm

rhodecode/model/permission.py

@@ -1,8 +1,15
-#!/usr/bin/env python
-# encoding: utf-8
-# Model for permissions
-# Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.model.permission
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
  
+    permissions model for RhodeCode
+    
+    :created_on: Aug 20, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
@@ -17,72 +24,91
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
-"""
-Created on Aug 20, 2010
-Model for permissions
-@author: marcink
-"""
 
-from pylons.i18n.translation import _
-from rhodecode.model.db import User, Permission, UserToPerm, RepoToPerm
-from rhodecode.model.meta import Session
 import logging
 import traceback
+
+from sqlalchemy.exc import DatabaseError
+
+from rhodecode.model import BaseModel
+from rhodecode.model.db import User, Permission, UserToPerm, RepoToPerm
+from rhodecode.model.caching_query import FromCache
+
 log = logging.getLogger(__name__)
 
 
-class PermissionModel(object):
+class PermissionModel(BaseModel):
+    """Permissions model for RhodeCode
+    """
 
-    def __init__(self):
-        self.sa = Session() 
-    
-    def get_default(self):
-        return self.sa.query(User).filter(User.username == 'default').scalar()
+    def get_permission(self, permission_id, cache=False):
+        """Get's permissions by id
     
-    def get_permission(self, id):
-        return self.sa.query(Permission).get(id)
+        :param permission_id: id of permission to get from database
+        :param cache: use Cache for this query
+        """
+        perm = self.sa.query(Permission)
+        if cache:
+            perm = perm.options(FromCache("sql_cache_short",
+                                          "get_permission_%s" % permission_id))
+        return perm.get(permission_id)
     
-    def get_permission_by_name(self, name):
-        return self.sa.query(Permission)\
-        .filter(Permission.permission_name == name).scalar()
+    def get_permission_by_name(self, name, cache=False):
+        """Get's permissions by given name
     
+        :param name: name to fetch
+        :param cache: Use cache for this query
+        """
+        perm = self.sa.query(Permission)\
+            .filter(Permission.permission_name == name)
+        if cache:
+            perm = perm.options(FromCache("sql_cache_short",
+                                          "get_permission_%s" % name))
+        return perm.scalar()
     
     def update(self, form_result):
         perm_user = self.sa.query(User)\
                 .filter(User.username == form_result['perm_user_name']).scalar()
         u2p = self.sa.query(UserToPerm).filter(UserToPerm.user == perm_user).all()
         if len(u2p) != 3:
-            raise Exception('There is more than 3 defined \
-            permissions for defualt user. This should not happen please verify\
-            your database')
+            raise Exception('Defined: %s should be 3  permissions for default'
+                            ' user. This should not happen please verify'
+                            ' your database' % len(u2p))
         
         try:
             #stage 1 change defaults    
             for p in u2p:
                 if p.permission.permission_name.startswith('repository.'):
-                    p.permission = self.get_permission_by_name(form_result['default_perm'])
+                    p.permission = self.get_permission_by_name(
+                                       form_result['default_perm'])
                     self.sa.add(p)
                     
                 if p.permission.permission_name.startswith('hg.register.'):
-                    p.permission = self.get_permission_by_name(form_result['default_register'])
+                    p.permission = self.get_permission_by_name(
+                                       form_result['default_register'])
                     self.sa.add(p)
                     
                 if p.permission.permission_name.startswith('hg.create.'):
-                    p.permission = self.get_permission_by_name(form_result['default_create'])
+                    p.permission = self.get_permission_by_name(
+                                        form_result['default_create'])
                     self.sa.add(p)
+
             #stage 2 update all default permissions for repos if checked
-            if form_result['overwrite_default'] == 'true':
-                for r2p in self.sa.query(RepoToPerm).filter(RepoToPerm.user == perm_user).all():
-                    r2p.permission = self.get_permission_by_name(form_result['default_perm'])
+            if form_result['overwrite_default'] == True:
+                for r2p in self.sa.query(RepoToPerm)\
+                               .filter(RepoToPerm.user == perm_user).all():
+                    r2p.permission = self.get_permission_by_name(
+                                         form_result['default_perm'])
                     self.sa.add(r2p)
             
+            #stage 3 set anonymous access
+            if perm_user.username == 'default':
+                perm_user.active = bool(form_result['anonymous'])
+                self.sa.add(perm_user)
+
+
             self.sa.commit()
-        except:
+        except (DatabaseError,):
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise               
-        
-        
-        
-        
-        

rhodecode/model/repo.py

@@ -1,7 +1,15
-#!/usr/bin/env python
-# encoding: utf-8
-# model for handling repositories actions
-# Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.model.repo
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Repository model for rhodecode
+    
+    :created_on: Jun 5, 2010
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
@@ -16,35 +24,44
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
-"""
-Created on Jun 5, 2010
-model for handling repositories actions
-@author: marcink
-"""
-from datetime import datetime
-from pylons import app_globals as g
-from rhodecode.lib.utils import check_repo
-from rhodecode.model.db import Repository, RepoToPerm, User, Permission
-from rhodecode.model.meta import Session
-from rhodecode.model.user_model import UserModel
-from rhodecode.lib.celerylib.tasks import create_repo_fork, run_task
-import logging
 import os
 import shutil
+import logging
 import traceback
+from datetime import datetime
+
+from pylons import app_globals as g
+
+from rhodecode.model import BaseModel
+from rhodecode.model.caching_query import FromCache
+from rhodecode.model.db import Repository, RepoToPerm, User, Permission, \
+    Statistics
+from rhodecode.model.user import UserModel
+
+from vcs.backends import get_backend
+
 log = logging.getLogger(__name__)
 
-class RepoModel(object):
+class RepoModel(BaseModel):
+
+    def get(self, repo_id, cache=False):
+        repo = self.sa.query(Repository)\
+            .filter(Repository.repo_id == repo_id)
     
-    def __init__(self, sa=None):
-        if not sa:
-            self.sa = Session()
-        else:
-            self.sa = sa
+        if cache:
+            repo = repo.options(FromCache("sql_cache_short",
+                                          "get_repo_%s" % repo_id))
+        return repo.scalar()
+
     
-    def get(self, id):
-        return self.sa.query(Repository)\
-            .filter(Repository.repo_name == id).scalar()
+    def get_by_repo_name(self, repo_name, cache=False):
+        repo = self.sa.query(Repository)\
+            .filter(Repository.repo_name == repo_name)
+
+        if cache:
+            repo = repo.options(FromCache("sql_cache_short",
+                                          "get_repo_%s" % repo_name))
+        return repo.scalar()
         
     def get_users_js(self):
         
@@ -58,38 +75,36 class RepoModel(object):
     
     def update(self, repo_name, form_data):
         try:
+            cur_repo = self.get_by_repo_name(repo_name, cache=False)
+            user_model = UserModel(self.sa)
 
             #update permissions
             for username, perm in form_data['perms_updates']:
                 r2p = self.sa.query(RepoToPerm)\
-                        .filter(RepoToPerm.user == self.sa.query(User)\
-                                .filter(User.username == username).one())\
-                        .filter(RepoToPerm.repository == self.get(repo_name))\
+                        .filter(RepoToPerm.user == user_model.get_by_username(username))\
+                        .filter(RepoToPerm.repository == cur_repo)\
                         .one()
                 
-                r2p.permission_id = self.sa.query(Permission).filter(
-                                                Permission.permission_name == 
-                                                perm).one().permission_id
+                r2p.permission = self.sa.query(Permission)\
+                                    .filter(Permission.permission_name == perm)\
+                                    .scalar()
                 self.sa.add(r2p)
             
             #set new permissions
             for username, perm in form_data['perms_new']:
                 r2p = RepoToPerm()
-                r2p.repository = self.get(repo_name)
-                r2p.user = self.sa.query(User)\
-                                .filter(User.username == username).one()
+                r2p.repository = cur_repo
+                r2p.user = user_model.get_by_username(username, cache=False)
                 
-                r2p.permission_id = self.sa.query(Permission).filter(
-                                        Permission.permission_name == perm)\
-                                        .one().permission_id
+                r2p.permission = self.sa.query(Permission)\
+                                    .filter(Permission.permission_name == perm)\
+                                    .scalar()
                 self.sa.add(r2p)
             
             #update current repo
-            cur_repo = self.get(repo_name)
-             
             for k, v in form_data.items():
                 if k == 'user':
-                    cur_repo.user_id = v
+                    cur_repo.user = user_model.get(v)
                 else:
                     setattr(cur_repo, k, v)
                                                         
@@ -108,12 +123,14 class RepoModel(object):
     def create(self, form_data, cur_user, just_db=False, fork=False):
         try:
             if fork:
+                #force str since hg doesn't go with unicode
                 repo_name = str(form_data['fork_name'])
                 org_name = str(form_data['repo_name'])
                 
             else:
                 org_name = repo_name = str(form_data['repo_name'])
             new_repo = Repository()
+            new_repo.enable_statistics = True
             for k, v in form_data.items():
                 if k == 'repo_name':
                     v = repo_name
@@ -130,7 +147,7 class RepoModel(object):
             #create default permission
             repo_to_perm = RepoToPerm()
             default = 'repository.read'
-            for p in UserModel(self.sa).get_default().user_perms:
+            for p in UserModel(self.sa).get_by_username('default', cache=False).user_perms:
                 if p.permission.permission_name.startswith('repository.'):
                     default = p.permission.permission_name
                     break
@@ -142,26 +159,34 class RepoModel(object):
                     .one().permission_id
                         
             repo_to_perm.repository_id = new_repo.repo_id
-            repo_to_perm.user_id = self.sa.query(User)\
-                    .filter(User.username == 'default').one().user_id 
+            repo_to_perm.user_id = UserModel(self.sa)\
+                .get_by_username('default', cache=False).user_id
             
             self.sa.add(repo_to_perm)
             self.sa.commit()
+
+
+            #now automatically start following this repository as owner
+            from rhodecode.model.scm import ScmModel
+            ScmModel(self.sa).toggle_following_repo(new_repo.repo_id,
+                                             cur_user.user_id)
+
             if not just_db:
-                self.__create_repo(repo_name)
+                self.__create_repo(repo_name, form_data['repo_type'])
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise    
     
     def create_fork(self, form_data, cur_user):
-        run_task(create_repo_fork, form_data, cur_user)
+        from rhodecode.lib.celerylib import tasks, run_task
+        run_task(tasks.create_repo_fork, form_data, cur_user)
                          
     def delete(self, repo):
         try:
             self.sa.delete(repo)
+            self.__delete_repo(repo)
             self.sa.commit()
-            self.__delete_repo(repo.repo_name)
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
@@ -170,7 +195,8 class RepoModel(object):
     def delete_perm_user(self, form_data, repo_name):
         try:
             self.sa.query(RepoToPerm)\
-                .filter(RepoToPerm.repository == self.get(repo_name))\
+                .filter(RepoToPerm.repository \
+                        == self.get_by_repo_name(repo_name))\
                 .filter(RepoToPerm.user_id == form_data['user_id']).delete()
             self.sa.commit()
         except:
@@ -178,14 +204,37 class RepoModel(object):
             self.sa.rollback()
             raise
            
-    def __create_repo(self, repo_name):        
+    def delete_stats(self, repo_name):
+        try:
+            self.sa.query(Statistics)\
+                .filter(Statistics.repository == \
+                        self.get_by_repo_name(repo_name)).delete()
+            self.sa.commit()
+        except:
+            log.error(traceback.format_exc())
+            self.sa.rollback()
+            raise
+
+
+    def __create_repo(self, repo_name, alias):
+        """
+        makes repository on filesystem
+        :param repo_name:
+        :param alias:
+        """
+        from rhodecode.lib.utils import check_repo
         repo_path = os.path.join(g.base_path, repo_name)
         if check_repo(repo_name, g.base_path):
             log.info('creating repo %s in %s', repo_name, repo_path)
-            from vcs.backends.hg import MercurialRepository
-            MercurialRepository(repo_path, create=True)
+            backend = get_backend(alias)
+            backend(repo_path, create=True)
 
     def __rename_repo(self, old, new):
+        """
+        renames repository on filesystem
+        :param old: old name
+        :param new: new name
+        """
         log.info('renaming repo from %s to %s', old, new)
         
         old_path = os.path.join(g.base_path, old)
@@ -195,11 +244,20 class RepoModel(object):
                             new_path)        
         shutil.move(old_path, new_path)
     
-    def __delete_repo(self, name):
-        rm_path = os.path.join(g.base_path, name)
+    def __delete_repo(self, repo):
+        """
+        removes repo from filesystem, the removal is acctually made by
+        added rm__ prefix into dir, and rename internat .hg/.git dirs so this
+        repository is no longer valid for rhodecode, can be undeleted later on
+        by reverting the renames on this repository
+        :param repo: repo object
+        """
+        rm_path = os.path.join(g.base_path, repo.repo_name)
         log.info("Removing %s", rm_path)
-        #disable hg 
-        shutil.move(os.path.join(rm_path, '.hg'), os.path.join(rm_path, 'rm__.hg'))
+        #disable hg/git
+        alias = repo.repo_type
+        shutil.move(os.path.join(rm_path, '.%s' % alias),
+                    os.path.join(rm_path, 'rm__.%s' % alias))
         #disable repo
         shutil.move(rm_path, os.path.join(g.base_path, 'rm__%s__%s' \
-                                          % (datetime.today(), name)))
+                                          % (datetime.today(), repo.repo_name)))