Added VCS into rhodecode core for faster and easier deployments of new versions

rhodecode/lib/vcs/__init__.py

@@ -0,0 +1,41 b''
+# -*- coding: utf-8 -*-
+"""
+    vcs
+    ~~~
+
+    Various version Control System (vcs) management abstraction layer for
+    Python.
+
+    :created_on: Apr 8, 2010
+    :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
+"""
+
+VERSION = (0, 2, 3, 'dev')
+
+__version__ = '.'.join((str(each) for each in VERSION[:4]))
+
+__all__ = [
+    'get_version', 'get_repo', 'get_backend',
+    'VCSError', 'RepositoryError', 'ChangesetError']
+
+import sys
+from rhodecode.lib.vcs.backends import get_repo, get_backend
+from rhodecode.lib.vcs.exceptions import VCSError, RepositoryError, ChangesetError
+
+
+def get_version():
+    """
+    Returns shorter version (digit parts only) as string.
+    """
+    return '.'.join((str(each) for each in VERSION[:3]))
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv
+    from rhodecode.lib.vcs.cli import ExecutionManager
+    manager = ExecutionManager(argv)
+    manager.execute()
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))

rhodecode/lib/vcs/backends/__init__.py

@@ -0,0 +1,63 b''
+# -*- coding: utf-8 -*-
+"""
+    vcs.backends
+    ~~~~~~~~~~~~
+
+    Main package for scm backends
+
+    :created_on: Apr 8, 2010
+    :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
+"""
+import os
+from pprint import pformat
+from rhodecode.lib.vcs.conf import settings
+from rhodecode.lib.vcs.exceptions import VCSError
+from rhodecode.lib.vcs.utils.helpers import get_scm
+from rhodecode.lib.vcs.utils.paths import abspath
+from rhodecode.lib.vcs.utils.imports import import_class
+
+
+def get_repo(path=None, alias=None, create=False):
+    """
+    Returns ``Repository`` object of type linked with given ``alias`` at
+    the specified ``path``. If ``alias`` is not given it will try to guess it
+    using get_scm method
+    """
+    if create:
+        if not (path or alias):
+            raise TypeError("If create is specified, we need path and scm type")
+        return get_backend(alias)(path, create=True)
+    if path is None:
+        path = abspath(os.path.curdir)
+    try:
+        scm, path = get_scm(path, search_recursively=True)
+        path = abspath(path)
+        alias = scm
+    except VCSError:
+        raise VCSError("No scm found at %s" % path)
+    if alias is None:
+        alias = get_scm(path)[0]
+
+    backend = get_backend(alias)
+    repo = backend(path, create=create)
+    return repo
+
+
+def get_backend(alias):
+    """
+    Returns ``Repository`` class identified by the given alias or raises
+    VCSError if alias is not recognized or backend class cannot be imported.
+    """
+    if alias not in settings.BACKENDS:
+        raise VCSError("Given alias '%s' is not recognized! Allowed aliases:\n"
+            "%s" % (alias, pformat(settings.BACKENDS.keys())))
+    backend_path = settings.BACKENDS[alias]
+    klass = import_class(backend_path)
+    return klass
+
+
+def get_supported_backends():
+    """
+    Returns list of aliases of supported backends.
+    """
+    return settings.BACKENDS.keys()

rhodecode/lib/vcs/backends/base.py

@@ -0,0 +1,911 b''
+# -*- coding: utf-8 -*-
+"""
+    vcs.backends.base
+    ~~~~~~~~~~~~~~~~~
+
+    Base for all available scm backends
+
+    :created_on: Apr 8, 2010
+    :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
+"""
+
+
+from itertools import chain
+from rhodecode.lib.vcs.utils import author_name, author_email
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs.utils.helpers import get_dict_for_attrs
+from rhodecode.lib.vcs.conf import settings
+
+from rhodecode.lib.vcs.exceptions import ChangesetError, EmptyRepositoryError, \
+    NodeAlreadyAddedError, NodeAlreadyChangedError, NodeAlreadyExistsError, \
+    NodeAlreadyRemovedError, NodeDoesNotExistError, NodeNotChangedError, \
+    RepositoryError
+
+
+class BaseRepository(object):
+    """
+    Base Repository for final backends
+
+    **Attributes**
+
+        ``DEFAULT_BRANCH_NAME``
+            name of default branch (i.e. "trunk" for svn, "master" for git etc.
+
+        ``scm``
+            alias of scm, i.e. *git* or *hg*
+
+        ``repo``
+            object from external api
+
+        ``revisions``
+            list of all available revisions' ids, in ascending order
+
+        ``changesets``
+            storage dict caching returned changesets
+
+        ``path``
+            absolute path to the repository
+
+        ``branches``
+            branches as list of changesets
+
+        ``tags``
+            tags as list of changesets
+    """
+    scm = None
+    DEFAULT_BRANCH_NAME = None
+    EMPTY_CHANGESET = '0' * 40
+
+    def __init__(self, repo_path, create=False, **kwargs):
+        """
+        Initializes repository. Raises RepositoryError if repository could
+        not be find at the given ``repo_path`` or directory at ``repo_path``
+        exists and ``create`` is set to True.
+
+        :param repo_path: local path of the repository
+        :param create=False: if set to True, would try to craete repository.
+        :param src_url=None: if set, should be proper url from which repository
+          would be cloned; requires ``create`` parameter to be set to True -
+          raises RepositoryError if src_url is set and create evaluates to
+          False
+        """
+        raise NotImplementedError
+
+    def __str__(self):
+        return '<%s at %s>' % (self.__class__.__name__, self.path)
+
+    def __repr__(self):
+        return self.__str__()
+
+    def __len__(self):
+        return self.count()
+
+    @LazyProperty
+    def alias(self):
+        for k, v in settings.BACKENDS.items():
+            if v.split('.')[-1] == str(self.__class__.__name__):
+                return k
+
+    @LazyProperty
+    def name(self):
+        raise NotImplementedError
+
+    @LazyProperty
+    def owner(self):
+        raise NotImplementedError
+
+    @LazyProperty
+    def description(self):
+        raise NotImplementedError
+
+    @LazyProperty
+    def size(self):
+        """
+        Returns combined size in bytes for all repository files
+        """
+
+        size = 0
+        try:
+            tip = self.get_changeset()
+            for topnode, dirs, files in tip.walk('/'):
+                for f in files:
+                    size += tip.get_file_size(f.path)
+                for dir in dirs:
+                    for f in files:
+                        size += tip.get_file_size(f.path)
+
+        except RepositoryError, e:
+            pass
+        return size
+
+    def is_valid(self):
+        """
+        Validates repository.
+        """
+        raise NotImplementedError
+
+    def get_last_change(self):
+        self.get_changesets()
+
+    #==========================================================================
+    # CHANGESETS
+    #==========================================================================
+
+    def get_changeset(self, revision=None):
+        """
+        Returns instance of ``Changeset`` class. If ``revision`` is None, most
+        recent changeset is returned.
+
+        :raises ``EmptyRepositoryError``: if there are no revisions
+        """
+        raise NotImplementedError
+
+    def __iter__(self):
+        """
+        Allows Repository objects to be iterated.
+
+        *Requires* implementation of ``__getitem__`` method.
+        """
+        for revision in self.revisions:
+            yield self.get_changeset(revision)
+
+    def get_changesets(self, start=None, end=None, start_date=None,
+                       end_date=None, branch_name=None, reverse=False):
+        """
+        Returns iterator of ``MercurialChangeset`` objects from start to end
+        not inclusive This should behave just like a list, ie. end is not
+        inclusive
+
+        :param start: None or str
+        :param end: None or str
+        :param start_date:
+        :param end_date:
+        :param branch_name:
+        :param reversed:
+        """
+        raise NotImplementedError
+
+    def __getslice__(self, i, j):
+        """
+        Returns a iterator of sliced repository
+        """
+        for rev in self.revisions[i:j]:
+            yield self.get_changeset(rev)
+
+    def __getitem__(self, key):
+        return self.get_changeset(key)
+
+    def count(self):
+        return len(self.revisions)
+
+    def tag(self, name, user, revision=None, message=None, date=None, **opts):
+        """
+        Creates and returns a tag for the given ``revision``.
+
+        :param name: name for new tag
+        :param user: full username, i.e.: "Joe Doe <[e-mail]>"
+        :param revision: changeset id for which new tag would be created
+        :param message: message of the tag's commit
+        :param date: date of tag's commit
+
+        :raises TagAlreadyExistError: if tag with same name already exists
+        """
+        raise NotImplementedError
+
+    def remove_tag(self, name, user, message=None, date=None):
+        """
+        Removes tag with the given ``name``.
+
+        :param name: name of the tag to be removed
+        :param user: full username, i.e.: "Joe Doe <[e-mail]>"
+        :param message: message of the tag's removal commit
+        :param date: date of tag's removal commit
+
+        :raises TagDoesNotExistError: if tag with given name does not exists
+        """
+        raise NotImplementedError
+
+    def get_diff(self, rev1, rev2, path=None, ignore_whitespace=False,
+            context=3):
+        """
+        Returns (git like) *diff*, as plain text. Shows changes introduced by
+        ``rev2`` since ``rev1``.
+
+        :param rev1: Entry point from which diff is shown. Can be
+          ``self.EMPTY_CHANGESET`` - in this case, patch showing all
+          the changes since empty state of the repository until ``rev2``
+        :param rev2: Until which revision changes should be shown.
+        :param ignore_whitespace: If set to ``True``, would not show whitespace
+          changes. Defaults to ``False``.
+        :param context: How many lines before/after changed lines should be
+          shown. Defaults to ``3``.
+        """
+        raise NotImplementedError
+
+    # ========== #
+    # COMMIT API #
+    # ========== #
+
+    @LazyProperty
+    def in_memory_changeset(self):
+        """
+        Returns ``InMemoryChangeset`` object for this repository.
+        """
+        raise NotImplementedError
+
+    def add(self, filenode, **kwargs):
+        """
+        Commit api function that will add given ``FileNode`` into this
+        repository.
+
+        :raises ``NodeAlreadyExistsError``: if there is a file with same path
+          already in repository
+        :raises ``NodeAlreadyAddedError``: if given node is already marked as
+          *added*
+        """
+        raise NotImplementedError
+
+    def remove(self, filenode, **kwargs):
+        """
+        Commit api function that will remove given ``FileNode`` into this
+        repository.
+
+        :raises ``EmptyRepositoryError``: if there are no changesets yet
+        :raises ``NodeDoesNotExistError``: if there is no file with given path
+        """
+        raise NotImplementedError
+
+    def commit(self, message, **kwargs):
+        """
+        Persists current changes made on this repository and returns newly
+        created changeset.
+
+        :raises ``NothingChangedError``: if no changes has been made
+        """
+        raise NotImplementedError
+
+    def get_state(self):
+        """
+        Returns dictionary with ``added``, ``changed`` and ``removed`` lists
+        containing ``FileNode`` objects.
+        """
+        raise NotImplementedError
+
+    def get_config_value(self, section, name, config_file=None):
+        """
+        Returns configuration value for a given [``section``] and ``name``.
+
+        :param section: Section we want to retrieve value from
+        :param name: Name of configuration we want to retrieve
+        :param config_file: A path to file which should be used to retrieve
+          configuration from (might also be a list of file paths)
+        """
+        raise NotImplementedError
+
+    def get_user_name(self, config_file=None):
+        """
+        Returns user's name from global configuration file.
+
+        :param config_file: A path to file which should be used to retrieve
+          configuration from (might also be a list of file paths)
+        """
+        raise NotImplementedError
+
+    def get_user_email(self, config_file=None):
+        """
+        Returns user's email from global configuration file.
+
+        :param config_file: A path to file which should be used to retrieve
+          configuration from (might also be a list of file paths)
+        """
+        raise NotImplementedError
+
+    # =========== #
+    # WORKDIR API #
+    # =========== #
+
+    @LazyProperty
+    def workdir(self):
+        """
+        Returns ``Workdir`` instance for this repository.
+        """
+        raise NotImplementedError
+
+
+class BaseChangeset(object):
+    """
+    Each backend should implement it's changeset representation.
+
+    **Attributes**
+
+        ``repository``
+            repository object within which changeset exists
+
+        ``id``
+            may be ``raw_id`` or i.e. for mercurial's tip just ``tip``
+
+        ``raw_id``
+            raw changeset representation (i.e. full 40 length sha for git
+            backend)
+
+        ``short_id``
+            shortened (if apply) version of ``raw_id``; it would be simple
+            shortcut for ``raw_id[:12]`` for git/mercurial backends or same
+            as ``raw_id`` for subversion
+
+        ``revision``
+            revision number as integer
+
+        ``files``
+            list of ``FileNode`` (``Node`` with NodeKind.FILE) objects
+
+        ``dirs``
+            list of ``DirNode`` (``Node`` with NodeKind.DIR) objects
+
+        ``nodes``
+            combined list of ``Node`` objects
+
+        ``author``
+            author of the changeset, as unicode
+
+        ``message``
+            message of the changeset, as unicode
+
+        ``parents``
+            list of parent changesets
+
+        ``last``
+            ``True`` if this is last changeset in repository, ``False``
+            otherwise; trying to access this attribute while there is no
+            changesets would raise ``EmptyRepositoryError``
+    """
+    def __str__(self):
+        return '<%s at %s:%s>' % (self.__class__.__name__, self.revision,
+            self.short_id)
+
+    def __repr__(self):
+        return self.__str__()
+
+    def __unicode__(self):
+        return u'%s:%s' % (self.revision, self.short_id)
+
+    def __eq__(self, other):
+        return self.raw_id == other.raw_id
+
+    @LazyProperty
+    def last(self):
+        if self.repository is None:
+            raise ChangesetError("Cannot check if it's most recent revision")
+        return self.raw_id == self.repository.revisions[-1]
+
+    @LazyProperty
+    def parents(self):
+        """
+        Returns list of parents changesets.
+        """
+        raise NotImplementedError
+
+    @LazyProperty
+    def id(self):
+        """
+        Returns string identifying this changeset.
+        """
+        raise NotImplementedError
+
+    @LazyProperty
+    def raw_id(self):
+        """
+        Returns raw string identifying this changeset.
+        """
+        raise NotImplementedError
+
+    @LazyProperty
+    def short_id(self):
+        """
+        Returns shortened version of ``raw_id`` attribute, as string,
+        identifying this changeset, useful for web representation.
+        """
+        raise NotImplementedError
+
+    @LazyProperty
+    def revision(self):
+        """
+        Returns integer identifying this changeset.
+
+        """
+        raise NotImplementedError
+
+    @LazyProperty
+    def author(self):
+        """
+        Returns Author for given commit
+        """
+
+        raise NotImplementedError
+
+    @LazyProperty
+    def author_name(self):
+        """
+        Returns Author name for given commit
+        """
+
+        return author_name(self.author)
+
+    @LazyProperty
+    def author_email(self):
+        """
+        Returns Author email address for given commit
+        """
+
+        return author_email(self.author)
+
+    def get_file_mode(self, path):
+        """
+        Returns stat mode of the file at the given ``path``.
+        """
+        raise NotImplementedError
+
+    def get_file_content(self, path):
+        """
+        Returns content of the file at the given ``path``.
+        """
+        raise NotImplementedError
+
+    def get_file_size(self, path):
+        """
+        Returns size of the file at the given ``path``.
+        """
+        raise NotImplementedError
+
+    def get_file_changeset(self, path):
+        """
+        Returns last commit of the file at the given ``path``.
+        """
+        raise NotImplementedError
+
+    def get_file_history(self, path):
+        """
+        Returns history of file as reversed list of ``Changeset`` objects for
+        which file at given ``path`` has been modified.
+        """
+        raise NotImplementedError
+
+    def get_nodes(self, path):
+        """
+        Returns combined ``DirNode`` and ``FileNode`` objects list representing
+        state of changeset at the given ``path``.
+
+        :raises ``ChangesetError``: if node at the given ``path`` is not
+          instance of ``DirNode``
+        """
+        raise NotImplementedError
+
+    def get_node(self, path):
+        """
+        Returns ``Node`` object from the given ``path``.
+
+        :raises ``NodeDoesNotExistError``: if there is no node at the given
+          ``path``
+        """
+        raise NotImplementedError
+
+    def fill_archive(self, stream=None, kind='tgz', prefix=None):
+        """
+        Fills up given stream.
+
+        :param stream: file like object.
+        :param kind: one of following: ``zip``, ``tar``, ``tgz``
+            or ``tbz2``. Default: ``tgz``.
+        :param prefix: name of root directory in archive.
+            Default is repository name and changeset's raw_id joined with dash.
+
+            repo-tip.<kind>
+        """
+
+        raise NotImplementedError
+
+    def get_chunked_archive(self, **kwargs):
+        """
+        Returns iterable archive. Tiny wrapper around ``fill_archive`` method.
+
+        :param chunk_size: extra parameter which controls size of returned
+            chunks. Default:8k.
+        """
+
+        chunk_size = kwargs.pop('chunk_size', 8192)
+        stream = kwargs.get('stream')
+        self.fill_archive(**kwargs)
+        while True:
+            data = stream.read(chunk_size)
+            if not data:
+                break
+            yield data
+
+    @LazyProperty
+    def root(self):
+        """
+        Returns ``RootNode`` object for this changeset.
+        """
+        return self.get_node('')
+
+    def next(self, branch=None):
+        """
+        Returns next changeset from current, if branch is gives it will return
+        next changeset belonging to this branch
+
+        :param branch: show changesets within the given named branch
+        """
+        raise NotImplementedError
+
+    def prev(self, branch=None):
+        """
+        Returns previous changeset from current, if branch is gives it will
+        return previous changeset belonging to this branch
+
+        :param branch: show changesets within the given named branch
+        """
+        raise NotImplementedError
+
+    @LazyProperty
+    def added(self):
+        """
+        Returns list of added ``FileNode`` objects.
+        """
+        raise NotImplementedError
+
+    @LazyProperty
+    def changed(self):
+        """
+        Returns list of modified ``FileNode`` objects.
+        """
+        raise NotImplementedError
+
+    @LazyProperty
+    def removed(self):
+        """
+        Returns list of removed ``FileNode`` objects.
+        """
+        raise NotImplementedError
+
+    @LazyProperty
+    def size(self):
+        """
+        Returns total number of bytes from contents of all filenodes.
+        """
+        return sum((node.size for node in self.get_filenodes_generator()))
+
+    def walk(self, topurl=''):
+        """
+        Similar to os.walk method. Insted of filesystem it walks through
+        changeset starting at given ``topurl``.  Returns generator of tuples
+        (topnode, dirnodes, filenodes).
+        """
+        topnode = self.get_node(topurl)
+        yield (topnode, topnode.dirs, topnode.files)
+        for dirnode in topnode.dirs:
+            for tup in self.walk(dirnode.path):
+                yield tup
+
+    def get_filenodes_generator(self):
+        """
+        Returns generator that yields *all* file nodes.
+        """
+        for topnode, dirs, files in self.walk():
+            for node in files:
+                yield node
+
+    def as_dict(self):
+        """
+        Returns dictionary with changeset's attributes and their values.
+        """
+        data = get_dict_for_attrs(self, ['id', 'raw_id', 'short_id',
+            'revision', 'date', 'message'])
+        data['author'] = {'name': self.author_name, 'email': self.author_email}
+        data['added'] = [node.path for node in self.added]
+        data['changed'] = [node.path for node in self.changed]
+        data['removed'] = [node.path for node in self.removed]
+        return data
+
+
+class BaseWorkdir(object):
+    """
+    Working directory representation of single repository.
+
+    :attribute: repository: repository object of working directory
+    """
+
+    def __init__(self, repository):
+        self.repository = repository
+
+    def get_branch(self):
+        """
+        Returns name of current branch.
+        """
+        raise NotImplementedError
+
+    def get_changeset(self):
+        """
+        Returns current changeset.
+        """
+        raise NotImplementedError
+
+    def get_added(self):
+        """
+        Returns list of ``FileNode`` objects marked as *new* in working
+        directory.
+        """
+        raise NotImplementedError
+
+    def get_changed(self):
+        """
+        Returns list of ``FileNode`` objects *changed* in working directory.
+        """
+        raise NotImplementedError
+
+    def get_removed(self):
+        """
+        Returns list of ``RemovedFileNode`` objects marked as *removed* in
+        working directory.
+        """
+        raise NotImplementedError
+
+    def get_untracked(self):
+        """
+        Returns list of ``FileNode`` objects which are present within working
+        directory however are not tracked by repository.
+        """
+        raise NotImplementedError
+
+    def get_status(self):
+        """
+        Returns dict with ``added``, ``changed``, ``removed`` and ``untracked``
+        lists.
+        """
+        raise NotImplementedError
+
+    def commit(self, message, **kwargs):
+        """
+        Commits local (from working directory) changes and returns newly
+        created
+        ``Changeset``. Updates repository's ``revisions`` list.
+
+        :raises ``CommitError``: if any error occurs while committing
+        """
+        raise NotImplementedError
+
+    def update(self, revision=None):
+        """
+        Fetches content of the given revision and populates it within working
+        directory.
+        """
+        raise NotImplementedError
+
+    def checkout_branch(self, branch=None):
+        """
+        Checks out ``branch`` or the backend's default branch.
+
+        Raises ``BranchDoesNotExistError`` if the branch does not exist.
+        """
+        raise NotImplementedError
+
+
+class BaseInMemoryChangeset(object):
+    """
+    Represents differences between repository's state (most recent head) and
+    changes made *in place*.
+
+    **Attributes**
+
+        ``repository``
+            repository object for this in-memory-changeset
+
+        ``added``
+            list of ``FileNode`` objects marked as *added*
+
+        ``changed``
+            list of ``FileNode`` objects marked as *changed*
+
+        ``removed``
+            list of ``FileNode`` or ``RemovedFileNode`` objects marked to be
+            *removed*
+
+        ``parents``
+            list of ``Changeset`` representing parents of in-memory changeset.
+            Should always be 2-element sequence.
+
+    """
+
+    def __init__(self, repository):
+        self.repository = repository
+        self.added = []
+        self.changed = []
+        self.removed = []
+        self.parents = []
+
+    def add(self, *filenodes):
+        """
+        Marks given ``FileNode`` objects as *to be committed*.
+
+        :raises ``NodeAlreadyExistsError``: if node with same path exists at
+          latest changeset
+        :raises ``NodeAlreadyAddedError``: if node with same path is already
+          marked as *added*
+        """
+        # Check if not already marked as *added* first
+        for node in filenodes:
+            if node.path in (n.path for n in self.added):
+                raise NodeAlreadyAddedError("Such FileNode %s is already "
+                    "marked for addition" % node.path)
+        for node in filenodes:
+            self.added.append(node)
+
+    def change(self, *filenodes):
+        """
+        Marks given ``FileNode`` objects to be *changed* in next commit.
+
+        :raises ``EmptyRepositoryError``: if there are no changesets yet
+        :raises ``NodeAlreadyExistsError``: if node with same path is already
+          marked to be *changed*
+        :raises ``NodeAlreadyRemovedError``: if node with same path is already
+          marked to be *removed*
+        :raises ``NodeDoesNotExistError``: if node doesn't exist in latest
+          changeset
+        :raises ``NodeNotChangedError``: if node hasn't really be changed
+        """
+        for node in filenodes:
+            if node.path in (n.path for n in self.removed):
+                raise NodeAlreadyRemovedError("Node at %s is already marked "
+                    "as removed" % node.path)
+        try:
+            self.repository.get_changeset()
+        except EmptyRepositoryError:
+            raise EmptyRepositoryError("Nothing to change - try to *add* new "
+                "nodes rather than changing them")
+        for node in filenodes:
+            if node.path in (n.path for n in self.changed):
+                raise NodeAlreadyChangedError("Node at '%s' is already "
+                    "marked as changed" % node.path)
+            self.changed.append(node)
+
+    def remove(self, *filenodes):
+        """
+        Marks given ``FileNode`` (or ``RemovedFileNode``) objects to be
+        *removed* in next commit.
+
+        :raises ``NodeAlreadyRemovedError``: if node has been already marked to
+          be *removed*
+        :raises ``NodeAlreadyChangedError``: if node has been already marked to
+          be *changed*
+        """
+        for node in filenodes:
+            if node.path in (n.path for n in self.removed):
+                raise NodeAlreadyRemovedError("Node is already marked to "
+                    "for removal at %s" % node.path)
+            if node.path in (n.path for n in self.changed):
+                raise NodeAlreadyChangedError("Node is already marked to "
+                    "be changed at %s" % node.path)
+            # We only mark node as *removed* - real removal is done by
+            # commit method
+            self.removed.append(node)
+
+    def reset(self):
+        """
+        Resets this instance to initial state (cleans ``added``, ``changed``
+        and ``removed`` lists).
+        """
+        self.added = []
+        self.changed = []
+        self.removed = []
+        self.parents = []
+
+    def get_ipaths(self):
+        """
+        Returns generator of paths from nodes marked as added, changed or
+        removed.
+        """
+        for node in chain(self.added, self.changed, self.removed):
+            yield node.path
+
+    def get_paths(self):
+        """
+        Returns list of paths from nodes marked as added, changed or removed.
+        """
+        return list(self.get_ipaths())
+
+    def check_integrity(self, parents=None):
+        """
+        Checks in-memory changeset's integrity. Also, sets parents if not
+        already set.
+
+        :raises CommitError: if any error occurs (i.e.
+          ``NodeDoesNotExistError``).
+        """
+        if not self.parents:
+            parents = parents or []
+            if len(parents) == 0:
+                try:
+                    parents = [self.repository.get_changeset(), None]
+                except EmptyRepositoryError:
+                    parents = [None, None]
+            elif len(parents) == 1:
+                parents += [None]
+            self.parents = parents
+
+        # Local parents, only if not None
+        parents = [p for p in self.parents if p]
+
+        # Check nodes marked as added
+        for p in parents:
+            for node in self.added:
+                try:
+                    p.get_node(node.path)
+                except NodeDoesNotExistError:
+                    pass
+                else:
+                    raise NodeAlreadyExistsError("Node at %s already exists "
+                        "at %s" % (node.path, p))
+
+        # Check nodes marked as changed
+        missing = set(self.changed)
+        not_changed = set(self.changed)
+        if self.changed and not parents:
+            raise NodeDoesNotExistError(str(self.changed[0].path))
+        for p in parents:
+            for node in self.changed:
+                try:
+                    old = p.get_node(node.path)
+                    missing.remove(node)
+                    if old.content != node.content:
+                        not_changed.remove(node)
+                except NodeDoesNotExistError:
+                    pass
+        if self.changed and missing:
+            raise NodeDoesNotExistError("Node at %s is missing "
+                "(parents: %s)" % (node.path, parents))
+
+        if self.changed and not_changed:
+            raise NodeNotChangedError("Node at %s wasn't actually changed "
+                "since parents' changesets: %s" % (not_changed.pop().path,
+                    parents)
+            )
+
+        # Check nodes marked as removed
+        if self.removed and not parents:
+            raise NodeDoesNotExistError("Cannot remove node at %s as there "
+                "were no parents specified" % self.removed[0].path)
+        really_removed = set()
+        for p in parents:
+            for node in self.removed:
+                try:
+                    p.get_node(node.path)
+                    really_removed.add(node)
+                except ChangesetError:
+                    pass
+        not_removed = set(self.removed) - really_removed
+        if not_removed:
+            raise NodeDoesNotExistError("Cannot remove node at %s from "
+                "following parents: %s" % (not_removed[0], parents))
+
+    def commit(self, message, author, parents=None, branch=None, date=None,
+            **kwargs):
+        """
+        Performs in-memory commit (doesn't check workdir in any way) and
+        returns newly created ``Changeset``. Updates repository's
+        ``revisions``.
+
+        .. note::
+            While overriding this method each backend's should call
+            ``self.check_integrity(parents)`` in the first place.
+
+        :param message: message of the commit
+        :param author: full username, i.e. "Joe Doe <[e-mail]>"
+        :param parents: single parent or sequence of parents from which commit
+          would be derieved
+        :param date: ``datetime.datetime`` instance. Defaults to
+          ``datetime.datetime.now()``.
+        :param branch: branch name, as string. If none given, default backend's
+          branch would be used.
+
+        :raises ``CommitError``: if any error occurs while committing
+        """
+        raise NotImplementedError

rhodecode/lib/vcs/backends/git/__init__.py

@@ -0,0 +1,9 b''
+from .repository import GitRepository
+from .changeset import GitChangeset
+from .inmemory import GitInMemoryChangeset
+from .workdir import GitWorkdir
+
+
+__all__ = [
+    'GitRepository', 'GitChangeset', 'GitInMemoryChangeset', 'GitWorkdir',
+]

rhodecode/lib/vcs/backends/git/changeset.py

@@ -0,0 +1,450 b''
+import re
+from itertools import chain
+from dulwich import objects
+from subprocess import Popen, PIPE
+from rhodecode.lib.vcs.conf import settings
+from rhodecode.lib.vcs.exceptions import RepositoryError
+from rhodecode.lib.vcs.exceptions import ChangesetError
+from rhodecode.lib.vcs.exceptions import NodeDoesNotExistError
+from rhodecode.lib.vcs.exceptions import VCSError
+from rhodecode.lib.vcs.exceptions import ChangesetDoesNotExistError
+from rhodecode.lib.vcs.exceptions import ImproperArchiveTypeError
+from rhodecode.lib.vcs.backends.base import BaseChangeset
+from rhodecode.lib.vcs.nodes import FileNode, DirNode, NodeKind, RootNode, RemovedFileNode
+from rhodecode.lib.vcs.utils import safe_unicode
+from rhodecode.lib.vcs.utils import date_fromtimestamp
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
+
+
+class GitChangeset(BaseChangeset):
+    """
+    Represents state of the repository at single revision.
+    """
+
+    def __init__(self, repository, revision):
+        self._stat_modes = {}
+        self.repository = repository
+        self.raw_id = revision
+        self.revision = repository.revisions.index(revision)
+
+        self.short_id = self.raw_id[:12]
+        self.id = self.raw_id
+        try:
+            commit = self.repository._repo.get_object(self.raw_id)
+        except KeyError:
+            raise RepositoryError("Cannot get object with id %s" % self.raw_id)
+        self._commit = commit
+        self._tree_id = commit.tree
+
+        try:
+            self.message = safe_unicode(commit.message[:-1])
+            # Always strip last eol
+        except UnicodeDecodeError:
+            self.message = commit.message[:-1].decode(commit.encoding
+                or 'utf-8')
+        #self.branch = None
+        self.tags = []
+        #tree = self.repository.get_object(self._tree_id)
+        self.nodes = {}
+        self._paths = {}
+
+    @LazyProperty
+    def author(self):
+        return safe_unicode(self._commit.committer)
+
+    @LazyProperty
+    def date(self):
+        return date_fromtimestamp(self._commit.commit_time,
+                                  self._commit.commit_timezone)
+
+    @LazyProperty
+    def status(self):
+        """
+        Returns modified, added, removed, deleted files for current changeset
+        """
+        return self.changed, self.added, self.removed
+
+    @LazyProperty
+    def branch(self):
+        # TODO: Cache as we walk (id <-> branch name mapping)
+        refs = self.repository._repo.get_refs()
+        heads = [(key[len('refs/heads/'):], val) for key, val in refs.items()
+            if key.startswith('refs/heads/')]
+
+        for name, id in heads:
+            walker = self.repository._repo.object_store.get_graph_walker([id])
+            while True:
+                id = walker.next()
+                if not id:
+                    break
+                if id == self.id:
+                    return safe_unicode(name)
+        raise ChangesetError("This should not happen... Have you manually "
+            "change id of the changeset?")
+
+    def _fix_path(self, path):
+        """
+        Paths are stored without trailing slash so we need to get rid off it if
+        needed.
+        """
+        if path.endswith('/'):
+            path = path.rstrip('/')
+        return path
+
+    def _get_id_for_path(self, path):
+        # FIXME: Please, spare a couple of minutes and make those codes cleaner;
+        if not path in self._paths:
+            path = path.strip('/')
+            # set root tree
+            tree = self.repository._repo[self._commit.tree]
+            if path == '':
+                self._paths[''] = tree.id
+                return tree.id
+            splitted = path.split('/')
+            dirs, name = splitted[:-1], splitted[-1]
+            curdir = ''
+            for dir in dirs:
+                if curdir:
+                    curdir = '/'.join((curdir, dir))
+                else:
+                    curdir = dir
+                #if curdir in self._paths:
+                    ## This path have been already traversed
+                    ## Update tree and continue
+                    #tree = self.repository._repo[self._paths[curdir]]
+                    #continue
+                dir_id = None
+                for item, stat, id in tree.iteritems():
+                    if curdir:
+                        item_path = '/'.join((curdir, item))
+                    else:
+                        item_path = item
+                    self._paths[item_path] = id
+                    self._stat_modes[item_path] = stat
+                    if dir == item:
+                        dir_id = id
+                if dir_id:
+                    # Update tree
+                    tree = self.repository._repo[dir_id]
+                    if not isinstance(tree, objects.Tree):
+                        raise ChangesetError('%s is not a directory' % curdir)
+                else:
+                    raise ChangesetError('%s have not been found' % curdir)
+            for item, stat, id in tree.iteritems():
+                if curdir:
+                    name = '/'.join((curdir, item))
+                else:
+                    name = item
+                self._paths[name] = id
+                self._stat_modes[name] = stat
+            if not path in self._paths:
+                raise NodeDoesNotExistError("There is no file nor directory "
+                    "at the given path %r at revision %r"
+                    % (path, self.short_id))
+        return self._paths[path]
+
+    def _get_kind(self, path):
+        id = self._get_id_for_path(path)
+        obj = self.repository._repo[id]
+        if isinstance(obj, objects.Blob):
+            return NodeKind.FILE
+        elif isinstance(obj, objects.Tree):
+            return NodeKind.DIR
+
+    def _get_file_nodes(self):
+        return chain(*(t[2] for t in self.walk()))
+
+    @LazyProperty
+    def parents(self):
+        """
+        Returns list of parents changesets.
+        """
+        return [self.repository.get_changeset(parent)
+            for parent in self._commit.parents]
+
+    def next(self, branch=None):
+
+        if branch and self.branch != branch:
+            raise VCSError('Branch option used on changeset not belonging '
+                           'to that branch')
+
+        def _next(changeset, branch):
+            try:
+                next_ = changeset.revision + 1
+                next_rev = changeset.repository.revisions[next_]
+            except IndexError:
+                raise ChangesetDoesNotExistError
+            cs = changeset.repository.get_changeset(next_rev)
+
+            if branch and branch != cs.branch:
+                return _next(cs, branch)
+
+            return cs
+
+        return _next(self, branch)
+
+    def prev(self, branch=None):
+        if branch and self.branch != branch:
+            raise VCSError('Branch option used on changeset not belonging '
+                           'to that branch')
+
+        def _prev(changeset, branch):
+            try:
+                prev_ = changeset.revision - 1
+                if prev_ < 0:
+                    raise IndexError
+                prev_rev = changeset.repository.revisions[prev_]
+            except IndexError:
+                raise ChangesetDoesNotExistError
+
+            cs = changeset.repository.get_changeset(prev_rev)
+
+            if branch and branch != cs.branch:
+                return _prev(cs, branch)
+
+            return cs
+
+        return _prev(self, branch)
+
+    def get_file_mode(self, path):
+        """
+        Returns stat mode of the file at the given ``path``.
+        """
+        # ensure path is traversed
+        self._get_id_for_path(path)
+        return self._stat_modes[path]
+
+    def get_file_content(self, path):
+        """
+        Returns content of the file at given ``path``.
+        """
+        id = self._get_id_for_path(path)
+        blob = self.repository._repo[id]
+        return blob.as_pretty_string()
+
+    def get_file_size(self, path):
+        """
+        Returns size of the file at given ``path``.
+        """
+        id = self._get_id_for_path(path)
+        blob = self.repository._repo[id]
+        return blob.raw_length()
+
+    def get_file_changeset(self, path):
+        """
+        Returns last commit of the file at the given ``path``.
+        """
+        node = self.get_node(path)
+        return node.history[0]
+
+    def get_file_history(self, path):
+        """
+        Returns history of file as reversed list of ``Changeset`` objects for
+        which file at given ``path`` has been modified.
+
+        TODO: This function now uses os underlying 'git' and 'grep' commands
+        which is generally not good. Should be replaced with algorithm
+        iterating commits.
+        """
+        cmd = 'log --name-status -p %s -- "%s" | grep "^commit"' \
+            % (self.id, path)
+        so, se = self.repository.run_git_command(cmd)
+        ids = re.findall(r'\w{40}', so)
+        return [self.repository.get_changeset(id) for id in ids]
+
+    def get_file_annotate(self, path):
+        """
+        Returns a list of three element tuples with lineno,changeset and line
+
+        TODO: This function now uses os underlying 'git' command which is
+        generally not good. Should be replaced with algorithm iterating
+        commits.
+        """
+        cmd = 'blame -l --root -r %s -- "%s"' % (self.id, path)
+        # -l     ==> outputs long shas (and we need all 40 characters)
+        # --root ==> doesn't put '^' character for bounderies
+        # -r sha ==> blames for the given revision
+        so, se = self.repository.run_git_command(cmd)
+        annotate = []
+        for i, blame_line in enumerate(so.split('\n')[:-1]):
+            ln_no = i + 1
+            id, line = re.split(r' \(.+?\) ', blame_line, 1)
+            annotate.append((ln_no, self.repository.get_changeset(id), line))
+        return annotate
+
+    def fill_archive(self, stream=None, kind='tgz', prefix=None,
+                     subrepos=False):
+        """
+        Fills up given stream.
+
+        :param stream: file like object.
+        :param kind: one of following: ``zip``, ``tgz`` or ``tbz2``.
+            Default: ``tgz``.
+        :param prefix: name of root directory in archive.
+            Default is repository name and changeset's raw_id joined with dash
+            (``repo-tip.<KIND>``).
+        :param subrepos: include subrepos in this archive.
+
+        :raise ImproperArchiveTypeError: If given kind is wrong.
+        :raise VcsError: If given stream is None
+
+        """
+        allowed_kinds = settings.ARCHIVE_SPECS.keys()
+        if kind not in allowed_kinds:
+            raise ImproperArchiveTypeError('Archive kind not supported use one'
+                'of %s', allowed_kinds)
+
+        if prefix is None:
+            prefix = '%s-%s' % (self.repository.name, self.short_id)
+        elif prefix.startswith('/'):
+            raise VCSError("Prefix cannot start with leading slash")
+        elif prefix.strip() == '':
+            raise VCSError("Prefix cannot be empty")
+
+        if kind == 'zip':
+            frmt = 'zip'
+        else:
+            frmt = 'tar'
+        cmd = 'git archive --format=%s --prefix=%s/ %s' % (frmt, prefix,
+            self.raw_id)
+        if kind == 'tgz':
+            cmd += ' | gzip -9'
+        elif kind == 'tbz2':
+            cmd += ' | bzip2 -9'
+
+        if stream is None:
+            raise VCSError('You need to pass in a valid stream for filling'
+                           ' with archival data')
+        popen = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True,
+            cwd=self.repository.path)
+
+        buffer_size = 1024 * 8
+        chunk = popen.stdout.read(buffer_size)
+        while chunk:
+            stream.write(chunk)
+            chunk = popen.stdout.read(buffer_size)
+        # Make sure all descriptors would be read
+        popen.communicate()
+
+    def get_nodes(self, path):
+        if self._get_kind(path) != NodeKind.DIR:
+            raise ChangesetError("Directory does not exist for revision %r at "
+                " %r" % (self.revision, path))
+        path = self._fix_path(path)
+        id = self._get_id_for_path(path)
+        tree = self.repository._repo[id]
+        dirnodes = []
+        filenodes = []
+        for name, stat, id in tree.iteritems():
+            obj = self.repository._repo.get_object(id)
+            if path != '':
+                obj_path = '/'.join((path, name))
+            else:
+                obj_path = name
+            if obj_path not in self._stat_modes:
+                self._stat_modes[obj_path] = stat
+            if isinstance(obj, objects.Tree):
+                dirnodes.append(DirNode(obj_path, changeset=self))
+            elif isinstance(obj, objects.Blob):
+                filenodes.append(FileNode(obj_path, changeset=self, mode=stat))
+            else:
+                raise ChangesetError("Requested object should be Tree "
+                                     "or Blob, is %r" % type(obj))
+        nodes = dirnodes + filenodes
+        for node in nodes:
+            if not node.path in self.nodes:
+                self.nodes[node.path] = node
+        nodes.sort()
+        return nodes
+
+    def get_node(self, path):
+        if isinstance(path, unicode):
+            path = path.encode('utf-8')
+        path = self._fix_path(path)
+        if not path in self.nodes:
+            try:
+                id = self._get_id_for_path(path)
+            except ChangesetError:
+                raise NodeDoesNotExistError("Cannot find one of parents' "
+                    "directories for a given path: %s" % path)
+            obj = self.repository._repo.get_object(id)
+            if isinstance(obj, objects.Tree):
+                if path == '':
+                    node = RootNode(changeset=self)
+                else:
+                    node = DirNode(path, changeset=self)
+                node._tree = obj
+            elif isinstance(obj, objects.Blob):
+                node = FileNode(path, changeset=self)
+                node._blob = obj
+            else:
+                raise NodeDoesNotExistError("There is no file nor directory "
+                    "at the given path %r at revision %r"
+                    % (path, self.short_id))
+            # cache node
+            self.nodes[path] = node
+        return self.nodes[path]
+
+    @LazyProperty
+    def affected_files(self):
+        """
+        Get's a fast accessible file changes for given changeset
+        """
+
+        return self.added + self.changed
+
+    @LazyProperty
+    def _diff_name_status(self):
+        output = []
+        for parent in self.parents:
+            cmd = 'diff --name-status %s %s' % (parent.raw_id, self.raw_id)
+            so, se = self.repository.run_git_command(cmd)
+            output.append(so.strip())
+        return '\n'.join(output)
+
+    def _get_paths_for_status(self, status):
+        """
+        Returns sorted list of paths for given ``status``.
+
+        :param status: one of: *added*, *modified* or *deleted*
+        """
+        paths = set()
+        char = status[0].upper()
+        for line in self._diff_name_status.splitlines():
+            if not line:
+                continue
+            if line.startswith(char):
+                splitted = line.split(char,1)
+                if not len(splitted) == 2:
+                    raise VCSError("Couldn't parse diff result:\n%s\n\n and "
+                        "particularly that line: %s" % (self._diff_name_status,
+                        line))
+                paths.add(splitted[1].strip())
+        return sorted(paths)
+
+    @LazyProperty
+    def added(self):
+        """
+        Returns list of added ``FileNode`` objects.
+        """
+        if not self.parents:
+            return list(self._get_file_nodes())
+        return [self.get_node(path) for path in self._get_paths_for_status('added')]
+
+    @LazyProperty
+    def changed(self):
+        """
+        Returns list of modified ``FileNode`` objects.
+        """
+        if not self.parents:
+            return []
+        return [self.get_node(path) for path in self._get_paths_for_status('modified')]
+
+    @LazyProperty
+    def removed(self):
+        """
+        Returns list of removed ``FileNode`` objects.
+        """
+        if not self.parents:
+            return []
+        return [RemovedFileNode(path) for path in self._get_paths_for_status('deleted')]

rhodecode/lib/vcs/backends/git/config.py

@@ -0,0 +1,347 b''
+# config.py - Reading and writing Git config files
+# Copyright (C) 2011 Jelmer Vernooij <[e-mail]>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your option) a later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+
+"""Reading and writing Git configuration files.
+
+TODO:
+ * preserve formatting when updating configuration files
+ * treat subsection names as case-insensitive for [branch.foo] style
+   subsections
+"""
+
+# Taken from dulwich not yet released 0.8.3 version (until it is actually
+# released)
+
+import errno
+import os
+import re
+
+from dulwich.file import GitFile
+
+
+class Config(object):
+    """A Git configuration."""
+
+    def get(self, section, name):
+        """Retrieve the contents of a configuration setting.
+
+        :param section: Tuple with section name and optional subsection namee
+        :param subsection: Subsection name
+        :return: Contents of the setting
+        :raise KeyError: if the value is not set
+        """
+        raise NotImplementedError(self.get)
+
+    def get_boolean(self, section, name, default=None):
+        """Retrieve a configuration setting as boolean.
+
+        :param section: Tuple with section name and optional subsection namee
+        :param name: Name of the setting, including section and possible
+            subsection.
+        :return: Contents of the setting
+        :raise KeyError: if the value is not set
+        """
+        try:
+            value = self.get(section, name)
+        except KeyError:
+            return default
+        if value.lower() == "true":
+            return True
+        elif value.lower() == "false":
+            return False
+        raise ValueError("not a valid boolean string: %r" % value)
+
+    def set(self, section, name, value):
+        """Set a configuration value.
+
+        :param name: Name of the configuration value, including section
+            and optional subsection
+        :param: Value of the setting
+        """
+        raise NotImplementedError(self.set)
+
+
+class ConfigDict(Config):
+    """Git configuration stored in a dictionary."""
+
+    def __init__(self, values=None):
+        """Create a new ConfigDict."""
+        if values is None:
+            values = {}
+        self._values = values
+
+    def __repr__(self):
+        return "%s(%r)" % (self.__class__.__name__, self._values)
+
+    def __eq__(self, other):
+        return (
+            isinstance(other, self.__class__) and
+            other._values == self._values)
+
+    @classmethod
+    def _parse_setting(cls, name):
+        parts = name.split(".")
+        if len(parts) == 3:
+            return (parts[0], parts[1], parts[2])
+        else:
+            return (parts[0], None, parts[1])
+
+    def get(self, section, name):
+        if isinstance(section, basestring):
+            section = (section, )
+        if len(section) > 1:
+            try:
+                return self._values[section][name]
+            except KeyError:
+                pass
+        return self._values[(section[0],)][name]
+
+    def set(self, section, name, value):
+        if isinstance(section, basestring):
+            section = (section, )
+        self._values.setdefault(section, {})[name] = value
+
+
+def _format_string(value):
+    if (value.startswith(" ") or
+        value.startswith("\t") or
+        value.endswith(" ") or
+        value.endswith("\t")):
+        return '"%s"' % _escape_value(value)
+    return _escape_value(value)
+
+
+def _parse_string(value):
+    value = value.strip()
+    ret = []
+    block = []
+    in_quotes = False
+    for c in value:
+        if c == "\"":
+            in_quotes = (not in_quotes)
+            ret.append(_unescape_value("".join(block)))
+            block = []
+        elif c in ("#", ";") and not in_quotes:
+            # the rest of the line is a comment
+            break
+        else:
+            block.append(c)
+
+    if in_quotes:
+        raise ValueError("value starts with quote but lacks end quote")
+
+    ret.append(_unescape_value("".join(block)).rstrip())
+
+    return "".join(ret)
+
+
+def _unescape_value(value):
+    """Unescape a value."""
+    def unescape(c):
+        return {
+            "\\\\": "\\",
+            "\\\"": "\"",
+            "\\n": "\n",
+            "\\t": "\t",
+            "\\b": "\b",
+            }[c.group(0)]
+    return re.sub(r"(\\.)", unescape, value)
+
+
+def _escape_value(value):
+    """Escape a value."""
+    return value.replace("\\", "\\\\").replace("\n", "\\n")\
+            .replace("\t", "\\t").replace("\"", "\\\"")
+
+
+def _check_variable_name(name):
+    for c in name:
+        if not c.isalnum() and c != '-':
+            return False
+    return True
+
+
+def _check_section_name(name):
+    for c in name:
+        if not c.isalnum() and c not in ('-', '.'):
+            return False
+    return True
+
+
+def _strip_comments(line):
+    line = line.split("#")[0]
+    line = line.split(";")[0]
+    return line
+
+
+class ConfigFile(ConfigDict):
+    """A Git configuration file, like .git/config or ~/.gitconfig.
+    """
+
+    @classmethod
+    def from_file(cls, f):
+        """Read configuration from a file-like object."""
+        ret = cls()
+        section = None
+        setting = None
+        for lineno, line in enumerate(f.readlines()):
+            line = line.lstrip()
+            if setting is None:
+                if _strip_comments(line).strip() == "":
+                    continue
+                if line[0] == "[":
+                    line = _strip_comments(line).rstrip()
+                    if line[-1] != "]":
+                        raise ValueError("expected trailing ]")
+                    key = line.strip()
+                    pts = key[1:-1].split(" ", 1)
+                    pts[0] = pts[0].lower()
+                    if len(pts) == 2:
+                        if pts[1][0] != "\"" or pts[1][-1] != "\"":
+                            raise ValueError(
+                                "Invalid subsection " + pts[1])
+                        else:
+                            pts[1] = pts[1][1:-1]
+                        if not _check_section_name(pts[0]):
+                            raise ValueError("invalid section name %s" %
+                                             pts[0])
+                        section = (pts[0], pts[1])
+                    else:
+                        if not _check_section_name(pts[0]):
+                            raise ValueError("invalid section name %s" %
+                                    pts[0])
+                        pts = pts[0].split(".", 1)
+                        if len(pts) == 2:
+                            section = (pts[0], pts[1])
+                        else:
+                            section = (pts[0], )
+                    ret._values[section] = {}
+                else:
+                    if section is None:
+                        raise ValueError("setting %r without section" % line)
+                    try:
+                        setting, value = line.split("=", 1)
+                    except ValueError:
+                        setting = line
+                        value = "true"
+                    setting = setting.strip().lower()
+                    if not _check_variable_name(setting):
+                        raise ValueError("invalid variable name %s" % setting)
+                    if value.endswith("\\\n"):
+                        value = value[:-2]
+                        continuation = True
+                    else:
+                        continuation = False
+                    value = _parse_string(value)
+                    ret._values[section][setting] = value
+                    if not continuation:
+                        setting = None
+            else:  # continuation line
+                if line.endswith("\\\n"):
+                    line = line[:-2]
+                    continuation = True
+                else:
+                    continuation = False
+                value = _parse_string(line)
+                ret._values[section][setting] += value
+                if not continuation:
+                    setting = None
+        return ret
+
+    @classmethod
+    def from_path(cls, path):
+        """Read configuration from a file on disk."""
+        f = GitFile(path, 'rb')
+        try:
+            ret = cls.from_file(f)
+            ret.path = path
+            return ret
+        finally:
+            f.close()
+
+    def write_to_path(self, path=None):
+        """Write configuration to a file on disk."""
+        if path is None:
+            path = self.path
+        f = GitFile(path, 'wb')
+        try:
+            self.write_to_file(f)
+        finally:
+            f.close()
+
+    def write_to_file(self, f):
+        """Write configuration to a file-like object."""
+        for section, values in self._values.iteritems():
+            try:
+                section_name, subsection_name = section
+            except ValueError:
+                (section_name, ) = section
+                subsection_name = None
+            if subsection_name is None:
+                f.write("[%s]\n" % section_name)
+            else:
+                f.write("[%s \"%s\"]\n" % (section_name, subsection_name))
+            for key, value in values.iteritems():
+                f.write("%s = %s\n" % (key, _escape_value(value)))
+
+
+class StackedConfig(Config):
+    """Configuration which reads from multiple config files.."""
+
+    def __init__(self, backends, writable=None):
+        self.backends = backends
+        self.writable = writable
+
+    def __repr__(self):
+        return "<%s for %r>" % (self.__class__.__name__, self.backends)
+
+    @classmethod
+    def default_backends(cls):
+        """Retrieve the default configuration.
+
+        This will look in the repository configuration (if for_path is
+        specified), the users' home directory and the system
+        configuration.
+        """
+        paths = []
+        paths.append(os.path.expanduser("~/.gitconfig"))
+        paths.append("/etc/gitconfig")
+        backends = []
+        for path in paths:
+            try:
+                cf = ConfigFile.from_path(path)
+            except (IOError, OSError), e:
+                if e.errno != errno.ENOENT:
+                    raise
+                else:
+                    continue
+            backends.append(cf)
+        return backends
+
+    def get(self, section, name):
+        for backend in self.backends:
+            try:
+                return backend.get(section, name)
+            except KeyError:
+                pass
+        raise KeyError(name)
+
+    def set(self, section, name, value):
+        if self.writable is None:
+            raise NotImplementedError(self.set)
+        return self.writable.set(section, name, value)

rhodecode/lib/vcs/backends/git/inmemory.py

@@ -0,0 +1,192 b''
+import time
+import datetime
+import posixpath
+from dulwich import objects
+from dulwich.repo import Repo
+from rhodecode.lib.vcs.backends.base import BaseInMemoryChangeset
+from rhodecode.lib.vcs.exceptions import RepositoryError
+
+
+class GitInMemoryChangeset(BaseInMemoryChangeset):
+
+    def commit(self, message, author, parents=None, branch=None, date=None,
+            **kwargs):
+        """
+        Performs in-memory commit (doesn't check workdir in any way) and
+        returns newly created ``Changeset``. Updates repository's
+        ``revisions``.
+
+        :param message: message of the commit
+        :param author: full username, i.e. "Joe Doe <[e-mail]>"
+        :param parents: single parent or sequence of parents from which commit
+          would be derieved
+        :param date: ``datetime.datetime`` instance. Defaults to
+          ``datetime.datetime.now()``.
+        :param branch: branch name, as string. If none given, default backend's
+          branch would be used.
+
+        :raises ``CommitError``: if any error occurs while committing
+        """
+        self.check_integrity(parents)
+
+        from .repository import GitRepository
+        if branch is None:
+            branch = GitRepository.DEFAULT_BRANCH_NAME
+
+        repo = self.repository._repo
+        object_store = repo.object_store
+
+        ENCODING = "UTF-8"
+        DIRMOD = 040000
+
+        # Create tree and populates it with blobs
+        commit_tree = self.parents[0] and repo[self.parents[0]._commit.tree] or\
+            objects.Tree()
+        for node in self.added + self.changed:
+            # Compute subdirs if needed
+            dirpath, nodename = posixpath.split(node.path)
+            dirnames = dirpath and dirpath.split('/') or []
+            parent = commit_tree
+            ancestors = [('', parent)]
+
+            # Tries to dig for the deepest existing tree
+            while dirnames:
+                curdir = dirnames.pop(0)
+                try:
+                    dir_id = parent[curdir][1]
+                except KeyError:
+                    # put curdir back into dirnames and stops
+                    dirnames.insert(0, curdir)
+                    break
+                else:
+                    # If found, updates parent
+                    parent = self.repository._repo[dir_id]
+                    ancestors.append((curdir, parent))
+            # Now parent is deepest exising tree and we need to create subtrees
+            # for dirnames (in reverse order) [this only applies for nodes from added]
+            new_trees = []
+            blob = objects.Blob.from_string(node.content.encode(ENCODING))
+            node_path = node.name.encode(ENCODING)
+            if dirnames:
+                # If there are trees which should be created we need to build
+                # them now (in reverse order)
+                reversed_dirnames = list(reversed(dirnames))
+                curtree = objects.Tree()
+                curtree[node_path] = node.mode, blob.id
+                new_trees.append(curtree)
+                for dirname in reversed_dirnames[:-1]:
+                    newtree = objects.Tree()
+                    #newtree.add(DIRMOD, dirname, curtree.id)
+                    newtree[dirname] = DIRMOD, curtree.id
+                    new_trees.append(newtree)
+                    curtree = newtree
+                parent[reversed_dirnames[-1]] = DIRMOD, curtree.id
+            else:
+                parent.add(node.mode, node_path, blob.id)
+            new_trees.append(parent)
+            # Update ancestors
+            for parent, tree, path in reversed([(a[1], b[1], b[0]) for a, b in
+                zip(ancestors, ancestors[1:])]):
+                parent[path] = DIRMOD, tree.id
+                object_store.add_object(tree)
+
+            object_store.add_object(blob)
+            for tree in new_trees:
+                object_store.add_object(tree)
+        for node in self.removed:
+            paths = node.path.split('/')
+            tree = commit_tree
+            trees = [tree]
+            # Traverse deep into the forest...
+            for path in paths:
+                try:
+                    obj = self.repository._repo[tree[path][1]]
+                    if isinstance(obj, objects.Tree):
+                        trees.append(obj)
+                        tree = obj
+                except KeyError:
+                    break
+            # Cut down the blob and all rotten trees on the way back...
+            for path, tree in reversed(zip(paths, trees)):
+                del tree[path]
+                if tree:
+                    # This tree still has elements - don't remove it or any
+                    # of it's parents
+                    break
+
+        object_store.add_object(commit_tree)
+
+        # Create commit
+        commit = objects.Commit()
+        commit.tree = commit_tree.id
+        commit.parents = [p._commit.id for p in self.parents if p]
+        commit.author = commit.committer = author
+        commit.encoding = ENCODING
+        commit.message = message + ' '
+
+        # Compute date
+        if date is None:
+            date = time.time()
+        elif isinstance(date, datetime.datetime):
+            date = time.mktime(date.timetuple())
+
+        author_time = kwargs.pop('author_time', date)
+        commit.commit_time = int(date)
+        commit.author_time = int(author_time)
+        tz = time.timezone
+        author_tz = kwargs.pop('author_timezone', tz)
+        commit.commit_timezone = tz
+        commit.author_timezone = author_tz
+
+        object_store.add_object(commit)
+
+        ref = 'refs/heads/%s' % branch
+        repo.refs[ref] = commit.id
+        repo.refs.set_symbolic_ref('HEAD', ref)
+
+        # Update vcs repository object & recreate dulwich repo
+        self.repository.revisions.append(commit.id)
+        self.repository._repo = Repo(self.repository.path)
+        tip = self.repository.get_changeset()
+        self.reset()
+        return tip
+
+    def _get_missing_trees(self, path, root_tree):
+        """
+        Creates missing ``Tree`` objects for the given path.
+
+        :param path: path given as a string. It may be a path to a file node
+          (i.e. ``foo/bar/baz.txt``) or directory path - in that case it must
+          end with slash (i.e. ``foo/bar/``).
+        :param root_tree: ``dulwich.objects.Tree`` object from which we start
+          traversing (should be commit's root tree)
+        """
+        dirpath = posixpath.split(path)[0]
+        dirs = dirpath.split('/')
+        if not dirs or dirs == ['']:
+            return []
+
+        def get_tree_for_dir(tree, dirname):
+            for name, mode, id in tree.iteritems():
+                if name == dirname:
+                    obj = self.repository._repo[id]
+                    if isinstance(obj, objects.Tree):
+                        return obj
+                    else:
+                        raise RepositoryError("Cannot create directory %s "
+                        "at tree %s as path is occupied and is not a "
+                        "Tree" % (dirname, tree))
+            return None
+
+        trees = []
+        parent = root_tree
+        for dirname in dirs:
+            tree = get_tree_for_dir(parent, dirname)
+            if tree is None:
+                tree = objects.Tree()
+                dirmode = 040000
+                parent.add(dirmode, dirname, tree.id)
+                parent = tree
+            # Always append tree
+            trees.append(tree)
+        return trees

rhodecode/lib/vcs/backends/git/repository.py

@@ -0,0 +1,508 b''
+# -*- coding: utf-8 -*-
+"""
+    vcs.backends.git
+    ~~~~~~~~~~~~~~~~
+
+    Git backend implementation.
+
+    :created_on: Apr 8, 2010
+    :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
+"""
+
+import os
+import re
+import time
+import posixpath
+from dulwich.repo import Repo, NotGitRepository
+#from dulwich.config import ConfigFile
+from string import Template
+from subprocess import Popen, PIPE
+from rhodecode.lib.vcs.backends.base import BaseRepository
+from rhodecode.lib.vcs.exceptions import BranchDoesNotExistError
+from rhodecode.lib.vcs.exceptions import ChangesetDoesNotExistError
+from rhodecode.lib.vcs.exceptions import EmptyRepositoryError
+from rhodecode.lib.vcs.exceptions import RepositoryError
+from rhodecode.lib.vcs.exceptions import TagAlreadyExistError
+from rhodecode.lib.vcs.exceptions import TagDoesNotExistError
+from rhodecode.lib.vcs.utils import safe_unicode, makedate, date_fromtimestamp
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs.utils.ordered_dict import OrderedDict
+from rhodecode.lib.vcs.utils.paths import abspath
+from rhodecode.lib.vcs.utils.paths import get_user_home
+from .workdir import GitWorkdir
+from .changeset import GitChangeset
+from .inmemory import GitInMemoryChangeset
+from .config import ConfigFile
+
+
+class GitRepository(BaseRepository):
+    """
+    Git repository backend.
+    """
+    DEFAULT_BRANCH_NAME = 'master'
+    scm = 'git'
+
+    def __init__(self, repo_path, create=False, src_url=None,
+                 update_after_clone=False, bare=False):
+
+        self.path = abspath(repo_path)
+        self._repo = self._get_repo(create, src_url, update_after_clone, bare)
+        try:
+            self.head = self._repo.head()
+        except KeyError:
+            self.head = None
+
+        self._config_files = [
+            bare and abspath(self.path, 'config') or abspath(self.path, '.git',
+                'config'),
+            abspath(get_user_home(), '.gitconfig'),
+        ]
+
+    @LazyProperty
+    def revisions(self):
+        """
+        Returns list of revisions' ids, in ascending order.  Being lazy
+        attribute allows external tools to inject shas from cache.
+        """
+        return self._get_all_revisions()
+
+    def run_git_command(self, cmd):
+        """
+        Runs given ``cmd`` as git command and returns tuple
+        (returncode, stdout, stderr).
+
+        .. note::
+           This method exists only until log/blame functionality is implemented
+           at Dulwich (see https://bugs.launchpad.net/bugs/645142). Parsing
+           os command's output is road to hell...
+
+        :param cmd: git command to be executed
+        """
+        #cmd = '(cd %s && git %s)' % (self.path, cmd)
+        if isinstance(cmd, basestring):
+            cmd = 'git %s' % cmd
+        else:
+            cmd = ['git'] + cmd
+        try:
+            opts = dict(
+                shell=isinstance(cmd, basestring),
+                stdout=PIPE,
+                stderr=PIPE)
+            if os.path.isdir(self.path):
+                opts['cwd'] = self.path
+            p = Popen(cmd, **opts)
+        except OSError, err:
+            raise RepositoryError("Couldn't run git command (%s).\n"
+                "Original error was:%s" % (cmd, err))
+        so, se = p.communicate()
+        if not se.startswith("fatal: bad default revision 'HEAD'") and \
+            p.returncode != 0:
+            raise RepositoryError("Couldn't run git command (%s).\n"
+                "stderr:\n%s" % (cmd, se))
+        return so, se
+
+    def _check_url(self, url):
+        """
+        Functon will check given url and try to verify if it's a valid
+        link. Sometimes it may happened that mercurial will issue basic
+        auth request that can cause whole API to hang when used from python
+        or other external calls.
+
+        On failures it'll raise urllib2.HTTPError
+        """
+
+        #TODO: implement this
+        pass
+
+    def _get_repo(self, create, src_url=None, update_after_clone=False,
+            bare=False):
+        if create and os.path.exists(self.path):
+            raise RepositoryError("Location already exist")
+        if src_url and not create:
+            raise RepositoryError("Create should be set to True if src_url is "
+                                  "given (clone operation creates repository)")
+        try:
+            if create and src_url:
+                self._check_url(src_url)
+                self.clone(src_url, update_after_clone, bare)
+                return Repo(self.path)
+            elif create:
+                os.mkdir(self.path)
+                if bare:
+                    return Repo.init_bare(self.path)
+                else:
+                    return Repo.init(self.path)
+            else:
+                return Repo(self.path)
+        except (NotGitRepository, OSError), err:
+            raise RepositoryError(err)
+
+    def _get_all_revisions(self):
+        cmd = 'rev-list --all --date-order'
+        try:
+            so, se = self.run_git_command(cmd)
+        except RepositoryError:
+            # Can be raised for empty repositories
+            return []
+        revisions = so.splitlines()
+        revisions.reverse()
+        return revisions
+
+    def _get_revision(self, revision):
+        """
+        For git backend we always return integer here. This way we ensure
+        that changset's revision attribute would become integer.
+        """
+        pattern = re.compile(r'^[[0-9a-fA-F]{12}|[0-9a-fA-F]{40}]$')
+        is_bstr = lambda o: isinstance(o, (str, unicode))
+        is_null = lambda o: len(o) == revision.count('0')
+
+        if len(self.revisions) == 0:
+            raise EmptyRepositoryError("There are no changesets yet")
+
+        if revision in (None, '', 'tip', 'HEAD', 'head', -1):
+            revision = self.revisions[-1]
+
+        if ((is_bstr(revision) and revision.isdigit() and len(revision) < 12)
+            or isinstance(revision, int) or is_null(revision)):
+            try:
+                revision = self.revisions[int(revision)]
+            except:
+                raise ChangesetDoesNotExistError("Revision %r does not exist "
+                    "for this repository %s" % (revision, self))
+
+        elif is_bstr(revision):
+            if not pattern.match(revision) or revision not in self.revisions:
+                raise ChangesetDoesNotExistError("Revision %r does not exist "
+                    "for this repository %s" % (revision, self))
+
+        # Ensure we return full id
+        if not pattern.match(str(revision)):
+            raise ChangesetDoesNotExistError("Given revision %r not recognized"
+                % revision)
+        return revision
+
+    def _get_archives(self, archive_name='tip'):
+
+        for i in [('zip', '.zip'), ('gz', '.tar.gz'), ('bz2', '.tar.bz2')]:
+                yield {"type": i[0], "extension": i[1], "node": archive_name}
+
+    def _get_url(self, url):
+        """
+        Returns normalized url. If schema is not given, would fall to
+        filesystem (``file:///``) schema.
+        """
+        url = str(url)
+        if url != 'default' and not '://' in url:
+            url = ':///'.join(('file', url))
+        return url
+
+    @LazyProperty
+    def name(self):
+        return os.path.basename(self.path)
+
+    @LazyProperty
+    def last_change(self):
+        """
+        Returns last change made on this repository as datetime object
+        """
+        return date_fromtimestamp(self._get_mtime(), makedate()[1])
+
+    def _get_mtime(self):
+        try:
+            return time.mktime(self.get_changeset().date.timetuple())
+        except RepositoryError:
+            # fallback to filesystem
+            in_path = os.path.join(self.path, '.git', "index")
+            he_path = os.path.join(self.path, '.git', "HEAD")
+            if os.path.exists(in_path):
+                return os.stat(in_path).st_mtime
+            else:
+                return os.stat(he_path).st_mtime
+
+    @LazyProperty
+    def description(self):
+        undefined_description = u'unknown'
+        description_path = os.path.join(self.path, '.git', 'description')
+        if os.path.isfile(description_path):
+            return safe_unicode(open(description_path).read())
+        else:
+            return undefined_description
+
+    @LazyProperty
+    def contact(self):
+        undefined_contact = u'Unknown'
+        return undefined_contact
+
+    @property
+    def branches(self):
+        if not self.revisions:
+            return {}
+        refs = self._repo.refs.as_dict()
+        sortkey = lambda ctx: ctx[0]
+        _branches = [('/'.join(ref.split('/')[2:]), head)
+            for ref, head in refs.items()
+            if ref.startswith('refs/heads/') or
+            ref.startswith('refs/remotes/') and not ref.endswith('/HEAD')]
+        return OrderedDict(sorted(_branches, key=sortkey, reverse=False))
+
+    def _get_tags(self):
+        if not self.revisions:
+            return {}
+        sortkey = lambda ctx: ctx[0]
+        _tags = [('/'.join(ref.split('/')[2:]), head) for ref, head in
+            self._repo.get_refs().items() if ref.startswith('refs/tags/')]
+        return OrderedDict(sorted(_tags, key=sortkey, reverse=True))
+
+    @LazyProperty
+    def tags(self):
+        return self._get_tags()
+
+    def tag(self, name, user, revision=None, message=None, date=None,
+            **kwargs):
+        """
+        Creates and returns a tag for the given ``revision``.
+
+        :param name: name for new tag
+        :param user: full username, i.e.: "Joe Doe <[e-mail]>"
+        :param revision: changeset id for which new tag would be created
+        :param message: message of the tag's commit
+        :param date: date of tag's commit
+
+        :raises TagAlreadyExistError: if tag with same name already exists
+        """
+        if name in self.tags:
+            raise TagAlreadyExistError("Tag %s already exists" % name)
+        changeset = self.get_changeset(revision)
+        message = message or "Added tag %s for commit %s" % (name,
+            changeset.raw_id)
+        self._repo.refs["refs/tags/%s" % name] = changeset._commit.id
+
+        self.tags = self._get_tags()
+        return changeset
+
+    def remove_tag(self, name, user, message=None, date=None):
+        """
+        Removes tag with the given ``name``.
+
+        :param name: name of the tag to be removed
+        :param user: full username, i.e.: "Joe Doe <[e-mail]>"
+        :param message: message of the tag's removal commit
+        :param date: date of tag's removal commit
+
+        :raises TagDoesNotExistError: if tag with given name does not exists
+        """
+        if name not in self.tags:
+            raise TagDoesNotExistError("Tag %s does not exist" % name)
+        tagpath = posixpath.join(self._repo.refs.path, 'refs', 'tags', name)
+        try:
+            os.remove(tagpath)
+            self.tags = self._get_tags()
+        except OSError, e:
+            raise RepositoryError(e.strerror)
+
+    def get_changeset(self, revision=None):
+        """
+        Returns ``GitChangeset`` object representing commit from git repository
+        at the given revision or head (most recent commit) if None given.
+        """
+        if isinstance(revision, GitChangeset):
+            return revision
+        revision = self._get_revision(revision)
+        changeset = GitChangeset(repository=self, revision=revision)
+        return changeset
+
+    def get_changesets(self, start=None, end=None, start_date=None,
+           end_date=None, branch_name=None, reverse=False):
+        """
+        Returns iterator of ``GitChangeset`` objects from start to end (both
+        are inclusive), in ascending date order (unless ``reverse`` is set).
+
+        :param start: changeset ID, as str; first returned changeset
+        :param end: changeset ID, as str; last returned changeset
+        :param start_date: if specified, changesets with commit date less than
+          ``start_date`` would be filtered out from returned set
+        :param end_date: if specified, changesets with commit date greater than
+          ``end_date`` would be filtered out from returned set
+        :param branch_name: if specified, changesets not reachable from given
+          branch would be filtered out from returned set
+        :param reverse: if ``True``, returned generator would be reversed
+          (meaning that returned changesets would have descending date order)
+
+        :raise BranchDoesNotExistError: If given ``branch_name`` does not
+            exist.
+        :raise ChangesetDoesNotExistError: If changeset for given ``start`` or
+          ``end`` could not be found.
+
+        """
+        if branch_name and branch_name not in self.branches:
+            raise BranchDoesNotExistError("Branch '%s' not found" \
+                                          % branch_name)
+        # %H at format means (full) commit hash, initial hashes are retrieved
+        # in ascending date order
+        cmd_template = 'log --date-order --reverse --pretty=format:"%H"'
+        cmd_params = {}
+        if start_date:
+            cmd_template += ' --since "$since"'
+            cmd_params['since'] = start_date.strftime('%m/%d/%y %H:%M:%S')
+        if end_date:
+            cmd_template += ' --until "$until"'
+            cmd_params['until'] = end_date.strftime('%m/%d/%y %H:%M:%S')
+        if branch_name:
+            cmd_template += ' $branch_name'
+            cmd_params['branch_name'] = branch_name
+        else:
+            cmd_template += ' --all'
+
+        cmd = Template(cmd_template).safe_substitute(**cmd_params)
+        revs = self.run_git_command(cmd)[0].splitlines()
+        start_pos = 0
+        end_pos = len(revs)
+        if start:
+            _start = self._get_revision(start)
+            try:
+                start_pos = revs.index(_start)
+            except ValueError:
+                pass
+
+        if end is not None:
+            _end = self._get_revision(end)
+            try:
+                end_pos = revs.index(_end)
+            except ValueError:
+                pass
+
+        if None not in [start, end] and start_pos > end_pos:
+            raise RepositoryError('start cannot be after end')
+
+        if end_pos is not None:
+            end_pos += 1
+
+        revs = revs[start_pos:end_pos]
+        if reverse:
+            revs = reversed(revs)
+        for rev in revs:
+            yield self.get_changeset(rev)
+
+    def get_diff(self, rev1, rev2, path=None, ignore_whitespace=False,
+            context=3):
+        """
+        Returns (git like) *diff*, as plain text. Shows changes introduced by
+        ``rev2`` since ``rev1``.
+
+        :param rev1: Entry point from which diff is shown. Can be
+          ``self.EMPTY_CHANGESET`` - in this case, patch showing all
+          the changes since empty state of the repository until ``rev2``
+        :param rev2: Until which revision changes should be shown.
+        :param ignore_whitespace: If set to ``True``, would not show whitespace
+          changes. Defaults to ``False``.
+        :param context: How many lines before/after changed lines should be
+          shown. Defaults to ``3``.
+        """
+        flags = ['-U%s' % context]
+        if ignore_whitespace:
+            flags.append('-w')
+
+        if rev1 == self.EMPTY_CHANGESET:
+            rev2 = self.get_changeset(rev2).raw_id
+            cmd = ' '.join(['show'] + flags + [rev2])
+        else:
+            rev1 = self.get_changeset(rev1).raw_id
+            rev2 = self.get_changeset(rev2).raw_id
+            cmd = ' '.join(['diff'] + flags + [rev1, rev2])
+
+        if path:
+            cmd += ' -- "%s"' % path
+        stdout, stderr = self.run_git_command(cmd)
+        # If we used 'show' command, strip first few lines (until actual diff
+        # starts)
+        if rev1 == self.EMPTY_CHANGESET:
+            lines = stdout.splitlines()
+            x = 0
+            for line in lines:
+                if line.startswith('diff'):
+                    break
+                x += 1
+            # Append new line just like 'diff' command do
+            stdout = '\n'.join(lines[x:]) + '\n'
+        return stdout
+
+    @LazyProperty
+    def in_memory_changeset(self):
+        """
+        Returns ``GitInMemoryChangeset`` object for this repository.
+        """
+        return GitInMemoryChangeset(self)
+
+    def clone(self, url, update_after_clone=True, bare=False):
+        """
+        Tries to clone changes from external location.
+
+        :param update_after_clone: If set to ``False``, git won't checkout
+          working directory
+        :param bare: If set to ``True``, repository would be cloned into
+          *bare* git repository (no working directory at all).
+        """
+        url = self._get_url(url)
+        cmd = ['clone']
+        if bare:
+            cmd.append('--bare')
+        elif not update_after_clone:
+            cmd.append('--no-checkout')
+        cmd += ['--', '"%s"' % url, '"%s"' % self.path]
+        cmd = ' '.join(cmd)
+        # If error occurs run_git_command raises RepositoryError already
+        self.run_git_command(cmd)
+
+    @LazyProperty
+    def workdir(self):
+        """
+        Returns ``Workdir`` instance for this repository.
+        """
+        return GitWorkdir(self)
+
+    def get_config_value(self, section, name, config_file=None):
+        """
+        Returns configuration value for a given [``section``] and ``name``.
+
+        :param section: Section we want to retrieve value from
+        :param name: Name of configuration we want to retrieve
+        :param config_file: A path to file which should be used to retrieve
+          configuration from (might also be a list of file paths)
+        """
+        if config_file is None:
+            config_file = []
+        elif isinstance(config_file, basestring):
+            config_file = [config_file]
+
+        def gen_configs():
+            for path in config_file + self._config_files:
+                try:
+                    yield ConfigFile.from_path(path)
+                except (IOError, OSError, ValueError):
+                    continue
+
+        for config in gen_configs():
+            try:
+                return config.get(section, name)
+            except KeyError:
+                continue
+        return None
+
+    def get_user_name(self, config_file=None):
+        """
+        Returns user's name from global configuration file.
+
+        :param config_file: A path to file which should be used to retrieve
+          configuration from (might also be a list of file paths)
+        """
+        return self.get_config_value('user', 'name', config_file)
+
+    def get_user_email(self, config_file=None):
+        """
+        Returns user's email from global configuration file.
+
+        :param config_file: A path to file which should be used to retrieve
+          configuration from (might also be a list of file paths)
+        """
+        return self.get_config_value('user', 'email', config_file)

rhodecode/lib/vcs/backends/git/workdir.py

@@ -0,0 +1,31 b''
+import re
+from rhodecode.lib.vcs.backends.base import BaseWorkdir
+from rhodecode.lib.vcs.exceptions import RepositoryError
+from rhodecode.lib.vcs.exceptions import BranchDoesNotExistError
+
+
+class GitWorkdir(BaseWorkdir):
+
+    def get_branch(self):
+        headpath = self.repository._repo.refs.refpath('HEAD')
+        try:
+            content = open(headpath).read()
+            match = re.match(r'^ref: refs/heads/(?P<branch>.+)\n$', content)
+            if match:
+                return match.groupdict()['branch']
+            else:
+                raise RepositoryError("Couldn't compute workdir's branch")
+        except IOError:
+            # Try naive way...
+            raise RepositoryError("Couldn't compute workdir's branch")
+
+    def get_changeset(self):
+        return self.repository.get_changeset(
+            self.repository._repo.refs.as_dict().get('HEAD'))
+
+    def checkout_branch(self, branch=None):
+        if branch is None:
+            branch = self.repository.DEFAULT_BRANCH_NAME
+        if branch not in self.repository.branches:
+            raise BranchDoesNotExistError
+        self.repository.run_git_command(['checkout', branch])

rhodecode/lib/vcs/backends/hg/__init__.py

@@ -0,0 +1,21 b''
+# -*- coding: utf-8 -*-
+"""
+    vcs.backends.hg
+    ~~~~~~~~~~~~~~~~
+
+    Mercurial backend implementation.
+
+    :created_on: Apr 8, 2010
+    :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
+"""
+
+from .repository import MercurialRepository
+from .changeset import MercurialChangeset
+from .inmemory import MercurialInMemoryChangeset
+from .workdir import MercurialWorkdir
+
+
+__all__ = [
+    'MercurialRepository', 'MercurialChangeset',
+    'MercurialInMemoryChangeset', 'MercurialWorkdir',
+]

rhodecode/lib/vcs/backends/hg/changeset.py

@@ -0,0 +1,338 b''
+import os
+import posixpath
+
+from rhodecode.lib.vcs.backends.base import BaseChangeset
+from rhodecode.lib.vcs.conf import settings
+from rhodecode.lib.vcs.exceptions import  ChangesetDoesNotExistError, \
+    ChangesetError, ImproperArchiveTypeError, NodeDoesNotExistError, VCSError
+from rhodecode.lib.vcs.nodes import AddedFileNodesGenerator, ChangedFileNodesGenerator, \
+    DirNode, FileNode, NodeKind, RemovedFileNodesGenerator, RootNode
+
+from rhodecode.lib.vcs.utils import safe_str, safe_unicode, date_fromtimestamp
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs.utils.paths import get_dirs_for_path
+
+from ...utils.hgcompat import archival, hex
+
+
+class MercurialChangeset(BaseChangeset):
+    """
+    Represents state of the repository at the single revision.
+    """
+
+    def __init__(self, repository, revision):
+        self.repository = repository
+        self.raw_id = revision
+        self._ctx = repository._repo[revision]
+        self.revision = self._ctx._rev
+        self.nodes = {}
+
+    @LazyProperty
+    def tags(self):
+        return map(safe_unicode, self._ctx.tags())
+
+    @LazyProperty
+    def branch(self):
+        return  safe_unicode(self._ctx.branch())
+
+    @LazyProperty
+    def message(self):
+        return safe_unicode(self._ctx.description())
+
+    @LazyProperty
+    def author(self):
+        return safe_unicode(self._ctx.user())
+
+    @LazyProperty
+    def date(self):
+        return date_fromtimestamp(*self._ctx.date())
+
+    @LazyProperty
+    def status(self):
+        """
+        Returns modified, added, removed, deleted files for current changeset
+        """
+        return self.repository._repo.status(self._ctx.p1().node(),
+                                            self._ctx.node())
+
+    @LazyProperty
+    def _file_paths(self):
+        return list(self._ctx)
+
+    @LazyProperty
+    def _dir_paths(self):
+        p = list(set(get_dirs_for_path(*self._file_paths)))
+        p.insert(0, '')
+        return p
+
+    @LazyProperty
+    def _paths(self):
+        return self._dir_paths + self._file_paths
+
+    @LazyProperty
+    def id(self):
+        if self.last:
+            return u'tip'
+        return self.short_id
+
+    @LazyProperty
+    def short_id(self):
+        return self.raw_id[:12]
+
+    @LazyProperty
+    def parents(self):
+        """
+        Returns list of parents changesets.
+        """
+        return [self.repository.get_changeset(parent.rev())
+                for parent in self._ctx.parents() if parent.rev() >= 0]
+
+    def next(self, branch=None):
+
+        if branch and self.branch != branch:
+            raise VCSError('Branch option used on changeset not belonging '
+                           'to that branch')
+
+        def _next(changeset, branch):
+            try:
+                next_ = changeset.revision + 1
+                next_rev = changeset.repository.revisions[next_]
+            except IndexError:
+                raise ChangesetDoesNotExistError
+            cs = changeset.repository.get_changeset(next_rev)
+
+            if branch and branch != cs.branch:
+                return _next(cs, branch)
+
+            return cs
+
+        return _next(self, branch)
+
+    def prev(self, branch=None):
+        if branch and self.branch != branch:
+            raise VCSError('Branch option used on changeset not belonging '
+                           'to that branch')
+
+        def _prev(changeset, branch):
+            try:
+                prev_ = changeset.revision - 1
+                if prev_ < 0:
+                    raise IndexError
+                prev_rev = changeset.repository.revisions[prev_]
+            except IndexError:
+                raise ChangesetDoesNotExistError
+
+            cs = changeset.repository.get_changeset(prev_rev)
+
+            if branch and branch != cs.branch:
+                return _prev(cs, branch)
+
+            return cs
+
+        return _prev(self, branch)
+
+    def _fix_path(self, path):
+        """
+        Paths are stored without trailing slash so we need to get rid off it if
+        needed. Also mercurial keeps filenodes as str so we need to decode
+        from unicode to str
+        """
+        if path.endswith('/'):
+            path = path.rstrip('/')
+
+        return safe_str(path)
+
+    def _get_kind(self, path):
+        path = self._fix_path(path)
+        if path in self._file_paths:
+            return NodeKind.FILE
+        elif path in self._dir_paths:
+            return NodeKind.DIR
+        else:
+            raise ChangesetError("Node does not exist at the given path %r"
+                % (path))
+
+    def _get_filectx(self, path):
+        path = self._fix_path(path)
+        if self._get_kind(path) != NodeKind.FILE:
+            raise ChangesetError("File does not exist for revision %r at "
+                " %r" % (self.revision, path))
+        return self._ctx.filectx(path)
+
+    def get_file_mode(self, path):
+        """
+        Returns stat mode of the file at the given ``path``.
+        """
+        fctx = self._get_filectx(path)
+        if 'x' in fctx.flags():
+            return 0100755
+        else:
+            return 0100644
+
+    def get_file_content(self, path):
+        """
+        Returns content of the file at given ``path``.
+        """
+        fctx = self._get_filectx(path)
+        return fctx.data()
+
+    def get_file_size(self, path):
+        """
+        Returns size of the file at given ``path``.
+        """
+        fctx = self._get_filectx(path)
+        return fctx.size()
+
+    def get_file_changeset(self, path):
+        """
+        Returns last commit of the file at the given ``path``.
+        """
+        fctx = self._get_filectx(path)
+        changeset = self.repository.get_changeset(fctx.linkrev())
+        return changeset
+
+    def get_file_history(self, path):
+        """
+        Returns history of file as reversed list of ``Changeset`` objects for
+        which file at given ``path`` has been modified.
+        """
+        fctx = self._get_filectx(path)
+        nodes = [fctx.filectx(x).node() for x in fctx.filelog()]
+        changesets = [self.repository.get_changeset(hex(node))
+            for node in reversed(nodes)]
+        return changesets
+
+    def get_file_annotate(self, path):
+        """
+        Returns a list of three element tuples with lineno,changeset and line
+        """
+        fctx = self._get_filectx(path)
+        annotate = []
+        for i, annotate_data in enumerate(fctx.annotate()):
+            ln_no = i + 1
+            annotate.append((ln_no, self.repository\
+                             .get_changeset(hex(annotate_data[0].node())),
+                             annotate_data[1],))
+
+        return annotate
+
+    def fill_archive(self, stream=None, kind='tgz', prefix=None,
+                     subrepos=False):
+        """
+        Fills up given stream.
+
+        :param stream: file like object.
+        :param kind: one of following: ``zip``, ``tgz`` or ``tbz2``.
+            Default: ``tgz``.
+        :param prefix: name of root directory in archive.
+            Default is repository name and changeset's raw_id joined with dash
+            (``repo-tip.<KIND>``).
+        :param subrepos: include subrepos in this archive.
+
+        :raise ImproperArchiveTypeError: If given kind is wrong.
+        :raise VcsError: If given stream is None
+        """
+
+        allowed_kinds = settings.ARCHIVE_SPECS.keys()
+        if kind not in allowed_kinds:
+            raise ImproperArchiveTypeError('Archive kind not supported use one'
+                'of %s', allowed_kinds)
+
+        if stream is None:
+            raise VCSError('You need to pass in a valid stream for filling'
+                           ' with archival data')
+
+        if prefix is None:
+            prefix = '%s-%s' % (self.repository.name, self.short_id)
+        elif prefix.startswith('/'):
+            raise VCSError("Prefix cannot start with leading slash")
+        elif prefix.strip() == '':
+            raise VCSError("Prefix cannot be empty")
+
+        archival.archive(self.repository._repo, stream, self.raw_id,
+                         kind, prefix=prefix, subrepos=subrepos)
+
+        #stream.close()
+
+        if stream.closed and hasattr(stream, 'name'):
+            stream = open(stream.name, 'rb')
+        elif hasattr(stream, 'mode') and 'r' not in stream.mode:
+            stream = open(stream.name, 'rb')
+        else:
+            stream.seek(0)
+
+    def get_nodes(self, path):
+        """
+        Returns combined ``DirNode`` and ``FileNode`` objects list representing
+        state of changeset at the given ``path``. If node at the given ``path``
+        is not instance of ``DirNode``, ChangesetError would be raised.
+        """
+
+        if self._get_kind(path) != NodeKind.DIR:
+            raise ChangesetError("Directory does not exist for revision %r at "
+                " %r" % (self.revision, path))
+        path = self._fix_path(path)
+        filenodes = [FileNode(f, changeset=self) for f in self._file_paths
+            if os.path.dirname(f) == path]
+        dirs = path == '' and '' or [d for d in self._dir_paths
+            if d and posixpath.dirname(d) == path]
+        dirnodes = [DirNode(d, changeset=self) for d in dirs
+            if os.path.dirname(d) == path]
+        nodes = dirnodes + filenodes
+        # cache nodes
+        for node in nodes:
+            self.nodes[node.path] = node
+        nodes.sort()
+        return nodes
+
+    def get_node(self, path):
+        """
+        Returns ``Node`` object from the given ``path``. If there is no node at
+        the given ``path``, ``ChangesetError`` would be raised.
+        """
+
+        path = self._fix_path(path)
+
+        if not path in self.nodes:
+            if path in self._file_paths:
+                node = FileNode(path, changeset=self)
+            elif path in self._dir_paths or path in self._dir_paths:
+                if path == '':
+                    node = RootNode(changeset=self)
+                else:
+                    node = DirNode(path, changeset=self)
+            else:
+                raise NodeDoesNotExistError("There is no file nor directory "
+                    "at the given path: %r at revision %r"
+                    % (path, self.short_id))
+            # cache node
+            self.nodes[path] = node
+        return self.nodes[path]
+
+    @LazyProperty
+    def affected_files(self):
+        """
+        Get's a fast accessible file changes for given changeset
+        """
+        return self._ctx.files()
+
+    @property
+    def added(self):
+        """
+        Returns list of added ``FileNode`` objects.
+        """
+        return AddedFileNodesGenerator([n for n in self.status[1]], self)
+
+    @property
+    def changed(self):
+        """
+        Returns list of modified ``FileNode`` objects.
+        """
+        return ChangedFileNodesGenerator([n for n in  self.status[0]], self)
+
+    @property
+    def removed(self):
+        """
+        Returns list of removed ``FileNode`` objects.
+        """
+        return RemovedFileNodesGenerator([n for n in self.status[2]], self)

rhodecode/lib/vcs/backends/hg/inmemory.py

@@ -0,0 +1,110 b''
+import datetime
+import errno
+
+from rhodecode.lib.vcs.backends.base import BaseInMemoryChangeset
+from rhodecode.lib.vcs.exceptions import RepositoryError
+
+from ...utils.hgcompat import memfilectx, memctx, hex
+
+
+class MercurialInMemoryChangeset(BaseInMemoryChangeset):
+
+    def commit(self, message, author, parents=None, branch=None, date=None,
+            **kwargs):
+        """
+        Performs in-memory commit (doesn't check workdir in any way) and
+        returns newly created ``Changeset``. Updates repository's
+        ``revisions``.
+
+        :param message: message of the commit
+        :param author: full username, i.e. "Joe Doe <[e-mail]>"
+        :param parents: single parent or sequence of parents from which commit
+          would be derieved
+        :param date: ``datetime.datetime`` instance. Defaults to
+          ``datetime.datetime.now()``.
+        :param branch: branch name, as string. If none given, default backend's
+          branch would be used.
+
+        :raises ``CommitError``: if any error occurs while committing
+        """
+        self.check_integrity(parents)
+
+        from .repository import MercurialRepository
+        if not isinstance(message, str) or not isinstance(author, str):
+            raise RepositoryError('Given message and author needs to be '
+                                  'an <str> instance')
+
+        if branch is None:
+            branch = MercurialRepository.DEFAULT_BRANCH_NAME
+        kwargs['branch'] = branch
+
+        def filectxfn(_repo, memctx, path):
+            """
+            Marks given path as added/changed/removed in a given _repo. This is
+            for internal mercurial commit function.
+            """
+
+            # check if this path is removed
+            if path in (node.path for node in self.removed):
+                # Raising exception is a way to mark node for removal
+                raise IOError(errno.ENOENT, '%s is deleted' % path)
+
+            # check if this path is added
+            for node in self.added:
+                if node.path == path:
+                    return memfilectx(path=node.path,
+                        data=(node.content.encode('utf8')
+                              if not node.is_binary else node.content),
+                        islink=False,
+                        isexec=node.is_executable,
+                        copied=False)
+
+            # or changed
+            for node in self.changed:
+                if node.path == path:
+                    return memfilectx(path=node.path,
+                        data=(node.content.encode('utf8')
+                              if not node.is_binary else node.content),
+                        islink=False,
+                        isexec=node.is_executable,
+                        copied=False)
+
+            raise RepositoryError("Given path haven't been marked as added,"
+                "changed or removed (%s)" % path)
+
+        parents = [None, None]
+        for i, parent in enumerate(self.parents):
+            if parent is not None:
+                parents[i] = parent._ctx.node()
+
+        if date and isinstance(date, datetime.datetime):
+            date = date.ctime()
+
+        commit_ctx = memctx(repo=self.repository._repo,
+            parents=parents,
+            text='',
+            files=self.get_paths(),
+            filectxfn=filectxfn,
+            user=author,
+            date=date,
+            extra=kwargs)
+
+        # injecting given _repo params
+        commit_ctx._text = message
+        commit_ctx._user = author
+        commit_ctx._date = date
+
+        # TODO: Catch exceptions!
+        n = self.repository._repo.commitctx(commit_ctx)
+        # Returns mercurial node
+        self._commit_ctx = commit_ctx  # For reference
+        # Update vcs repository object & recreate mercurial _repo
+        # new_ctx = self.repository._repo[node]
+        # new_tip = self.repository.get_changeset(new_ctx.hex())
+        new_id = hex(n)
+        self.repository.revisions.append(new_id)
+        self._repo = self.repository._get_repo(create=False)
+        self.repository.branches = self.repository._get_branches()
+        tip = self.repository.get_changeset()
+        self.reset()
+        return tip

rhodecode/lib/vcs/backends/hg/repository.py

@@ -0,0 +1,521 b''
+import os
+import time
+import datetime
+import urllib
+import urllib2
+
+from rhodecode.lib.vcs.backends.base import BaseRepository
+from .workdir import MercurialWorkdir
+from .changeset import MercurialChangeset
+from .inmemory import MercurialInMemoryChangeset
+
+from rhodecode.lib.vcs.exceptions import BranchDoesNotExistError, \
+    ChangesetDoesNotExistError, EmptyRepositoryError, RepositoryError, \
+    VCSError, TagAlreadyExistError, TagDoesNotExistError
+from rhodecode.lib.vcs.utils import author_email, author_name, date_fromtimestamp, \
+    makedate, safe_unicode
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs.utils.ordered_dict import OrderedDict
+from rhodecode.lib.vcs.utils.paths import abspath
+
+from ...utils.hgcompat import ui, nullid, match, patch, diffopts, clone, \
+    get_contact, pull, localrepository, RepoLookupError, Abort, RepoError, hex
+
+
+class MercurialRepository(BaseRepository):
+    """
+    Mercurial repository backend
+    """
+    DEFAULT_BRANCH_NAME = 'default'
+    scm = 'hg'
+
+    def __init__(self, repo_path, create=False, baseui=None, src_url=None,
+                 update_after_clone=False):
+        """
+        Raises RepositoryError if repository could not be find at the given
+        ``repo_path``.
+
+        :param repo_path: local path of the repository
+        :param create=False: if set to True, would try to create repository if
+           it does not exist rather than raising exception
+        :param baseui=None: user data
+        :param src_url=None: would try to clone repository from given location
+        :param update_after_clone=False: sets update of working copy after
+          making a clone
+        """
+
+        if not isinstance(repo_path, str):
+            raise VCSError('Mercurial backend requires repository path to '
+                           'be instance of <str> got %s instead' %
+                           type(repo_path))
+
+        self.path = abspath(repo_path)
+        self.baseui = baseui or ui.ui()
+        # We've set path and ui, now we can set _repo itself
+        self._repo = self._get_repo(create, src_url, update_after_clone)
+
+    @property
+    def _empty(self):
+        """
+        Checks if repository is empty without any changesets
+        """
+        # TODO: Following raises errors when using InMemoryChangeset...
+        # return len(self._repo.changelog) == 0
+        return len(self.revisions) == 0
+
+    @LazyProperty
+    def revisions(self):
+        """
+        Returns list of revisions' ids, in ascending order.  Being lazy
+        attribute allows external tools to inject shas from cache.
+        """
+        return self._get_all_revisions()
+
+    @LazyProperty
+    def name(self):
+        return os.path.basename(self.path)
+
+    @LazyProperty
+    def branches(self):
+        return self._get_branches()
+
+    def _get_branches(self, closed=False):
+        """
+        Get's branches for this repository
+        Returns only not closed branches by default
+
+        :param closed: return also closed branches for mercurial
+        """
+
+        if self._empty:
+            return {}
+
+        def _branchtags(localrepo):
+            """
+            Patched version of mercurial branchtags to not return the closed
+            branches
+
+            :param localrepo: locarepository instance
+            """
+
+            bt = {}
+            bt_closed = {}
+            for bn, heads in localrepo.branchmap().iteritems():
+                tip = heads[-1]
+                if 'close' in localrepo.changelog.read(tip)[5]:
+                    bt_closed[bn] = tip
+                else:
+                    bt[bn] = tip
+
+            if closed:
+                bt.update(bt_closed)
+            return bt
+
+        sortkey = lambda ctx: ctx[0]  # sort by name
+        _branches = [(safe_unicode(n), hex(h),) for n, h in
+                     _branchtags(self._repo).items()]
+
+        return OrderedDict(sorted(_branches, key=sortkey, reverse=False))
+
+    @LazyProperty
+    def tags(self):
+        """
+        Get's tags for this repository
+        """
+        return self._get_tags()
+
+    def _get_tags(self):
+        if self._empty:
+            return {}
+
+        sortkey = lambda ctx: ctx[0]  # sort by name
+        _tags = [(safe_unicode(n), hex(h),) for n, h in
+                 self._repo.tags().items()]
+
+        return OrderedDict(sorted(_tags, key=sortkey, reverse=True))
+
+    def tag(self, name, user, revision=None, message=None, date=None,
+            **kwargs):
+        """
+        Creates and returns a tag for the given ``revision``.
+
+        :param name: name for new tag
+        :param user: full username, i.e.: "Joe Doe <[e-mail]>"
+        :param revision: changeset id for which new tag would be created
+        :param message: message of the tag's commit
+        :param date: date of tag's commit
+
+        :raises TagAlreadyExistError: if tag with same name already exists
+        """
+        if name in self.tags:
+            raise TagAlreadyExistError("Tag %s already exists" % name)
+        changeset = self.get_changeset(revision)
+        local = kwargs.setdefault('local', False)
+
+        if message is None:
+            message = "Added tag %s for changeset %s" % (name,
+                changeset.short_id)
+
+        if date is None:
+            date = datetime.datetime.now().ctime()
+
+        try:
+            self._repo.tag(name, changeset._ctx.node(), message, local, user,
+                date)
+        except Abort, e:
+            raise RepositoryError(e.message)
+
+        # Reinitialize tags
+        self.tags = self._get_tags()
+        tag_id = self.tags[name]
+
+        return self.get_changeset(revision=tag_id)
+
+    def remove_tag(self, name, user, message=None, date=None):
+        """
+        Removes tag with the given ``name``.
+
+        :param name: name of the tag to be removed
+        :param user: full username, i.e.: "Joe Doe <[e-mail]>"
+        :param message: message of the tag's removal commit
+        :param date: date of tag's removal commit
+
+        :raises TagDoesNotExistError: if tag with given name does not exists
+        """
+        if name not in self.tags:
+            raise TagDoesNotExistError("Tag %s does not exist" % name)
+        if message is None:
+            message = "Removed tag %s" % name
+        if date is None:
+            date = datetime.datetime.now().ctime()
+        local = False
+
+        try:
+            self._repo.tag(name, nullid, message, local, user, date)
+            self.tags = self._get_tags()
+        except Abort, e:
+            raise RepositoryError(e.message)
+
+    @LazyProperty
+    def bookmarks(self):
+        """
+        Get's bookmarks for this repository
+        """
+        return self._get_bookmarks()
+
+    def _get_bookmarks(self):
+        if self._empty:
+            return {}
+
+        sortkey = lambda ctx: ctx[0]  # sort by name
+        _bookmarks = [(safe_unicode(n), hex(h),) for n, h in
+                 self._repo._bookmarks.items()]
+        return OrderedDict(sorted(_bookmarks, key=sortkey, reverse=True))
+
+    def _get_all_revisions(self):
+
+        return map(lambda x: hex(x[7]), self._repo.changelog.index)[:-1]
+
+    def get_diff(self, rev1, rev2, path='', ignore_whitespace=False,
+                  context=3):
+        """
+        Returns (git like) *diff*, as plain text. Shows changes introduced by
+        ``rev2`` since ``rev1``.
+
+        :param rev1: Entry point from which diff is shown. Can be
+          ``self.EMPTY_CHANGESET`` - in this case, patch showing all
+          the changes since empty state of the repository until ``rev2``
+        :param rev2: Until which revision changes should be shown.
+        :param ignore_whitespace: If set to ``True``, would not show whitespace
+          changes. Defaults to ``False``.
+        :param context: How many lines before/after changed lines should be
+          shown. Defaults to ``3``.
+        """
+        # Check if given revisions are present at repository (may raise
+        # ChangesetDoesNotExistError)
+        if rev1 != self.EMPTY_CHANGESET:
+            self.get_changeset(rev1)
+        self.get_changeset(rev2)
+
+        file_filter = match(self.path, '', [path])
+        return ''.join(patch.diff(self._repo, rev1, rev2, match=file_filter,
+                          opts=diffopts(git=True,
+                                        ignorews=ignore_whitespace,
+                                        context=context)))
+
+    def _check_url(self, url):
+        """
+        Function will check given url and try to verify if it's a valid
+        link. Sometimes it may happened that mercurial will issue basic
+        auth request that can cause whole API to hang when used from python
+        or other external calls.
+
+        On failures it'll raise urllib2.HTTPError, return code 200 if url
+        is valid or True if it's a local path
+        """
+
+        from mercurial.util import url as Url
+
+        # those authnadlers are patched for python 2.6.5 bug an
+        # infinit looping when given invalid resources
+        from mercurial.url import httpbasicauthhandler, httpdigestauthhandler
+
+        # check first if it's not an local url
+        if os.path.isdir(url) or url.startswith('file:'):
+            return True
+
+        handlers = []
+        test_uri, authinfo = Url(url).authinfo()
+
+        if authinfo:
+            #create a password manager
+            passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+            passmgr.add_password(*authinfo)
+
+            handlers.extend((httpbasicauthhandler(passmgr),
+                             httpdigestauthhandler(passmgr)))
+
+        o = urllib2.build_opener(*handlers)
+        o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
+                        ('Accept', 'application/mercurial-0.1')]
+
+        q = {"cmd": 'between'}
+        q.update({'pairs': "%s-%s" % ('0' * 40, '0' * 40)})
+        qs = '?%s' % urllib.urlencode(q)
+        cu = "%s%s" % (test_uri, qs)
+        req = urllib2.Request(cu, None, {})
+
+        try:
+            resp = o.open(req)
+            return resp.code == 200
+        except Exception, e:
+            # means it cannot be cloned
+            raise urllib2.URLError(e)
+
+    def _get_repo(self, create, src_url=None, update_after_clone=False):
+        """
+        Function will check for mercurial repository in given path and return
+        a localrepo object. If there is no repository in that path it will
+        raise an exception unless ``create`` parameter is set to True - in
+        that case repository would be created and returned.
+        If ``src_url`` is given, would try to clone repository from the
+        location at given clone_point. Additionally it'll make update to
+        working copy accordingly to ``update_after_clone`` flag
+        """
+        try:
+            if src_url:
+                url = str(self._get_url(src_url))
+                opts = {}
+                if not update_after_clone:
+                    opts.update({'noupdate': True})
+                try:
+                    self._check_url(url)
+                    clone(self.baseui, url, self.path, **opts)
+#                except urllib2.URLError:
+#                    raise Abort("Got HTTP 404 error")
+                except Exception:
+                    raise
+                # Don't try to create if we've already cloned repo
+                create = False
+            return localrepository(self.baseui, self.path, create=create)
+        except (Abort, RepoError), err:
+            if create:
+                msg = "Cannot create repository at %s. Original error was %s"\
+                    % (self.path, err)
+            else:
+                msg = "Not valid repository at %s. Original error was %s"\
+                    % (self.path, err)
+            raise RepositoryError(msg)
+
+    @LazyProperty
+    def in_memory_changeset(self):
+        return MercurialInMemoryChangeset(self)
+
+    @LazyProperty
+    def description(self):
+        undefined_description = u'unknown'
+        return safe_unicode(self._repo.ui.config('web', 'description',
+                                   undefined_description, untrusted=True))
+
+    @LazyProperty
+    def contact(self):
+        undefined_contact = u'Unknown'
+        return safe_unicode(get_contact(self._repo.ui.config)
+                            or undefined_contact)
+
+    @LazyProperty
+    def last_change(self):
+        """
+        Returns last change made on this repository as datetime object
+        """
+        return date_fromtimestamp(self._get_mtime(), makedate()[1])
+
+    def _get_mtime(self):
+        try:
+            return time.mktime(self.get_changeset().date.timetuple())
+        except RepositoryError:
+            #fallback to filesystem
+            cl_path = os.path.join(self.path, '.hg', "00changelog.i")
+            st_path = os.path.join(self.path, '.hg', "store")
+            if os.path.exists(cl_path):
+                return os.stat(cl_path).st_mtime
+            else:
+                return os.stat(st_path).st_mtime
+
+    def _get_hidden(self):
+        return self._repo.ui.configbool("web", "hidden", untrusted=True)
+
+    def _get_revision(self, revision):
+        """
+        Get's an ID revision given as str. This will always return a fill
+        40 char revision number
+
+        :param revision: str or int or None
+        """
+
+        if self._empty:
+            raise EmptyRepositoryError("There are no changesets yet")
+
+        if revision in [-1, 'tip', None]:
+            revision = 'tip'
+
+        try:
+            revision = hex(self._repo.lookup(revision))
+        except (IndexError, ValueError, RepoLookupError, TypeError):
+            raise ChangesetDoesNotExistError("Revision %r does not "
+                                    "exist for this repository %s" \
+                                    % (revision, self))
+        return revision
+
+    def _get_archives(self, archive_name='tip'):
+        allowed = self.baseui.configlist("web", "allow_archive",
+                                         untrusted=True)
+        for i in [('zip', '.zip'), ('gz', '.tar.gz'), ('bz2', '.tar.bz2')]:
+            if i[0] in allowed or self._repo.ui.configbool("web",
+                                                           "allow" + i[0],
+                                                           untrusted=True):
+                yield {"type": i[0], "extension": i[1], "node": archive_name}
+
+    def _get_url(self, url):
+        """
+        Returns normalized url. If schema is not given, would fall
+        to filesystem
+        (``file:///``) schema.
+        """
+        url = str(url)
+        if url != 'default' and not '://' in url:
+            url = "file:" + urllib.pathname2url(url)
+        return url
+
+    def get_changeset(self, revision=None):
+        """
+        Returns ``MercurialChangeset`` object representing repository's
+        changeset at the given ``revision``.
+        """
+        revision = self._get_revision(revision)
+        changeset = MercurialChangeset(repository=self, revision=revision)
+        return changeset
+
+    def get_changesets(self, start=None, end=None, start_date=None,
+                       end_date=None, branch_name=None, reverse=False):
+        """
+        Returns iterator of ``MercurialChangeset`` objects from start to end
+        (both are inclusive)
+
+        :param start: None, str, int or mercurial lookup format
+        :param end:  None, str, int or mercurial lookup format
+        :param start_date:
+        :param end_date:
+        :param branch_name:
+        :param reversed: return changesets in reversed order
+        """
+
+        start_raw_id = self._get_revision(start)
+        start_pos = self.revisions.index(start_raw_id) if start else None
+        end_raw_id = self._get_revision(end)
+        end_pos = self.revisions.index(end_raw_id) if end else None
+
+        if None not in [start, end] and start_pos > end_pos:
+            raise RepositoryError("start revision '%s' cannot be "
+                                  "after end revision '%s'" % (start, end))
+
+        if branch_name and branch_name not in self.branches.keys():
+            raise BranchDoesNotExistError('Such branch %s does not exists for'
+                                  ' this repository' % branch_name)
+        if end_pos is not None:
+            end_pos += 1
+
+        slice_ = reversed(self.revisions[start_pos:end_pos]) if reverse else \
+            self.revisions[start_pos:end_pos]
+
+        for id_ in slice_:
+            cs = self.get_changeset(id_)
+            if branch_name and cs.branch != branch_name:
+                continue
+            if start_date and cs.date < start_date:
+                continue
+            if end_date and cs.date > end_date:
+                continue
+
+            yield cs
+
+    def pull(self, url):
+        """
+        Tries to pull changes from external location.
+        """
+        url = self._get_url(url)
+        try:
+            pull(self.baseui, self._repo, url)
+        except Abort, err:
+            # Propagate error but with vcs's type
+            raise RepositoryError(str(err))
+
+    @LazyProperty
+    def workdir(self):
+        """
+        Returns ``Workdir`` instance for this repository.
+        """
+        return MercurialWorkdir(self)
+
+    def get_config_value(self, section, name, config_file=None):
+        """
+        Returns configuration value for a given [``section``] and ``name``.
+
+        :param section: Section we want to retrieve value from
+        :param name: Name of configuration we want to retrieve
+        :param config_file: A path to file which should be used to retrieve
+          configuration from (might also be a list of file paths)
+        """
+        if config_file is None:
+            config_file = []
+        elif isinstance(config_file, basestring):
+            config_file = [config_file]
+
+        config = self._repo.ui
+        for path in config_file:
+            config.readconfig(path)
+        return config.config(section, name)
+
+    def get_user_name(self, config_file=None):
+        """
+        Returns user's name from global configuration file.
+
+        :param config_file: A path to file which should be used to retrieve
+          configuration from (might also be a list of file paths)
+        """
+        username = self.get_config_value('ui', 'username')
+        if username:
+            return author_name(username)
+        return None
+
+    def get_user_email(self, config_file=None):
+        """
+        Returns user's email from global configuration file.
+
+        :param config_file: A path to file which should be used to retrieve
+          configuration from (might also be a list of file paths)
+        """
+        username = self.get_config_value('ui', 'username')
+        if username:
+            return author_email(username)
+        return None

rhodecode/lib/vcs/backends/hg/workdir.py

@@ -0,0 +1,21 b''
+from rhodecode.lib.vcs.backends.base import BaseWorkdir
+from rhodecode.lib.vcs.exceptions import BranchDoesNotExistError
+
+from ...utils.hgcompat import hg_merge
+
+
+class MercurialWorkdir(BaseWorkdir):
+
+    def get_branch(self):
+        return self.repository._repo.dirstate.branch()
+
+    def get_changeset(self):
+        return self.repository.get_changeset()
+
+    def checkout_branch(self, branch=None):
+        if branch is None:
+            branch = self.repository.DEFAULT_BRANCH_NAME
+        if branch not in self.repository.branches:
+            raise BranchDoesNotExistError
+
+        hg_merge.update(self.repository._repo, branch, False, False, None)

rhodecode/lib/vcs/conf/settings.py

@@ -0,0 +1,33 b''
+import os
+import tempfile
+from rhodecode.lib.vcs.utils.paths import get_user_home
+
+abspath = lambda * p: os.path.abspath(os.path.join(*p))
+
+VCSRC_PATH = os.environ.get('VCSRC_PATH')
+
+if not VCSRC_PATH:
+    HOME_ = get_user_home()
+    if not HOME_:
+        HOME_ = tempfile.gettempdir()
+
+VCSRC_PATH = VCSRC_PATH or abspath(HOME_, '.vcsrc')
+if os.path.isdir(VCSRC_PATH):
+    VCSRC_PATH = os.path.join(VCSRC_PATH, '__init__.py')
+
+BACKENDS = {
+    'hg': 'vcs.backends.hg.MercurialRepository',
+    'git': 'vcs.backends.git.GitRepository',
+}
+
+ARCHIVE_SPECS = {
+    'tar': ('application/x-tar', '.tar'),
+    'tbz2': ('application/x-bzip2', '.tar.bz2'),
+    'tgz': ('application/x-gzip', '.tar.gz'),
+    'zip': ('application/zip', '.zip'),
+}
+
+BACKENDS = {
+    'hg': 'rhodecode.lib.vcs.backends.hg.MercurialRepository',
+    'git': 'rhodecode.lib.vcs.backends.git.GitRepository',
+}

rhodecode/lib/vcs/exceptions.py

@@ -0,0 +1,93 b''
+# -*- coding: utf-8 -*-
+"""
+    vcs.exceptions
+    ~~~~~~~~~~~~~~
+
+    Custom exceptions module
+
+    :created_on: Apr 8, 2010
+    :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
+"""
+
+
+class VCSError(Exception):
+    pass
+
+
+class RepositoryError(VCSError):
+    pass
+
+
+class EmptyRepositoryError(RepositoryError):
+    pass
+
+
+class TagAlreadyExistError(RepositoryError):
+    pass
+
+
+class TagDoesNotExistError(RepositoryError):
+    pass
+
+
+class BranchAlreadyExistError(RepositoryError):
+    pass
+
+
+class BranchDoesNotExistError(RepositoryError):
+    pass
+
+
+class ChangesetError(RepositoryError):
+    pass
+
+
+class ChangesetDoesNotExistError(ChangesetError):
+    pass
+
+
+class CommitError(RepositoryError):
+    pass
+
+
+class NothingChangedError(CommitError):
+    pass
+
+
+class NodeError(VCSError):
+    pass
+
+
+class RemovedFileNodeError(NodeError):
+    pass
+
+
+class NodeAlreadyExistsError(CommitError):
+    pass
+
+
+class NodeAlreadyChangedError(CommitError):
+    pass
+
+
+class NodeDoesNotExistError(CommitError):
+    pass
+
+
+class NodeNotChangedError(CommitError):
+    pass
+
+
+class NodeAlreadyAddedError(CommitError):
+    pass
+
+
+class NodeAlreadyRemovedError(CommitError):
+    pass
+
+
+class ImproperArchiveTypeError(VCSError):
+    pass
+
+class CommandError(VCSError):
+    pass

rhodecode/lib/vcs/nodes.py

@@ -0,0 +1,551 b''
+# -*- coding: utf-8 -*-
+"""
+    vcs.nodes
+    ~~~~~~~~~
+
+    Module holding everything related to vcs nodes.
+
+    :created_on: Apr 8, 2010
+    :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
+"""
+import stat
+import posixpath
+import mimetypes
+
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs.utils import safe_unicode
+from rhodecode.lib.vcs.exceptions import NodeError
+from rhodecode.lib.vcs.exceptions import RemovedFileNodeError
+
+from pygments import lexers
+
+
+class NodeKind:
+    DIR = 1
+    FILE = 2
+
+
+class NodeState:
+    ADDED = u'added'
+    CHANGED = u'changed'
+    NOT_CHANGED = u'not changed'
+    REMOVED = u'removed'
+
+
+class NodeGeneratorBase(object):
+    """
+    Base class for removed added and changed filenodes, it's a lazy generator
+    class that will create filenodes only on iteration or call
+
+    The len method doesn't need to create filenodes at all
+    """
+
+    def __init__(self, current_paths, cs):
+        self.cs = cs
+        self.current_paths = current_paths
+
+    def __call__(self):
+        return [n for n in self]
+
+    def __getslice__(self, i, j):
+        for p in self.current_paths[i:j]:
+            yield self.cs.get_node(p)
+
+    def __len__(self):
+        return len(self.current_paths)
+
+    def __iter__(self):
+        for p in self.current_paths:
+            yield self.cs.get_node(p)
+
+
+class AddedFileNodesGenerator(NodeGeneratorBase):
+    """
+    Class holding Added files for current changeset
+    """
+    pass
+
+
+class ChangedFileNodesGenerator(NodeGeneratorBase):
+    """
+    Class holding Changed files for current changeset
+    """
+    pass
+
+
+class RemovedFileNodesGenerator(NodeGeneratorBase):
+    """
+    Class holding removed files for current changeset
+    """
+    def __iter__(self):
+        for p in self.current_paths:
+            yield RemovedFileNode(path=p)
+
+    def __getslice__(self, i, j):
+        for p in self.current_paths[i:j]:
+            yield RemovedFileNode(path=p)
+
+
+class Node(object):
+    """
+    Simplest class representing file or directory on repository.  SCM backends
+    should use ``FileNode`` and ``DirNode`` subclasses rather than ``Node``
+    directly.
+
+    Node's ``path`` cannot start with slash as we operate on *relative* paths
+    only. Moreover, every single node is identified by the ``path`` attribute,
+    so it cannot end with slash, too. Otherwise, path could lead to mistakes.
+    """
+
+    def __init__(self, path, kind):
+        if path.startswith('/'):
+            raise NodeError("Cannot initialize Node objects with slash at "
+                "the beginning as only relative paths are supported")
+        self.path = path.rstrip('/')
+        if path == '' and kind != NodeKind.DIR:
+            raise NodeError("Only DirNode and its subclasses may be "
+                            "initialized with empty path")
+        self.kind = kind
+        #self.dirs, self.files = [], []
+        if self.is_root() and not self.is_dir():
+            raise NodeError("Root node cannot be FILE kind")
+
+    @LazyProperty
+    def parent(self):
+        parent_path = self.get_parent_path()
+        if parent_path:
+            if self.changeset:
+                return self.changeset.get_node(parent_path)
+            return DirNode(parent_path)
+        return None
+
+    @LazyProperty
+    def name(self):
+        """
+        Returns name of the node so if its path
+        then only last part is returned.
+        """
+        return safe_unicode(self.path.rstrip('/').split('/')[-1])
+
+    def _get_kind(self):
+        return self._kind
+
+    def _set_kind(self, kind):
+        if hasattr(self, '_kind'):
+            raise NodeError("Cannot change node's kind")
+        else:
+            self._kind = kind
+            # Post setter check (path's trailing slash)
+            if self.path.endswith('/'):
+                raise NodeError("Node's path cannot end with slash")
+
+    kind = property(_get_kind, _set_kind)
+
+    def __cmp__(self, other):
+        """
+        Comparator using name of the node, needed for quick list sorting.
+        """
+        kind_cmp = cmp(self.kind, other.kind)
+        if kind_cmp:
+            return kind_cmp
+        return cmp(self.name, other.name)
+
+    def __eq__(self, other):
+        for attr in ['name', 'path', 'kind']:
+            if getattr(self, attr) != getattr(other, attr):
+                return False
+        if self.is_file():
+            if self.content != other.content:
+                return False
+        else:
+            # For DirNode's check without entering each dir
+            self_nodes_paths = list(sorted(n.path for n in self.nodes))
+            other_nodes_paths = list(sorted(n.path for n in self.nodes))
+            if self_nodes_paths != other_nodes_paths:
+                return False
+        return True
+
+    def __nq__(self, other):
+        return not self.__eq__(other)
+
+    def __repr__(self):
+        return '<%s %r>' % (self.__class__.__name__, self.path)
+
+    def __str__(self):
+        return self.__repr__()
+
+    def __unicode__(self):
+        return self.name
+
+    def get_parent_path(self):
+        """
+        Returns node's parent path or empty string if node is root.
+        """
+        if self.is_root():
+            return ''
+        return posixpath.dirname(self.path.rstrip('/')) + '/'
+
+    def is_file(self):
+        """
+        Returns ``True`` if node's kind is ``NodeKind.FILE``, ``False``
+        otherwise.
+        """
+        return self.kind == NodeKind.FILE
+
+    def is_dir(self):
+        """
+        Returns ``True`` if node's kind is ``NodeKind.DIR``, ``False``
+        otherwise.
+        """
+        return self.kind == NodeKind.DIR
+
+    def is_root(self):
+        """
+        Returns ``True`` if node is a root node and ``False`` otherwise.
+        """
+        return self.kind == NodeKind.DIR and self.path == ''
+
+    @LazyProperty
+    def added(self):
+        return self.state is NodeState.ADDED
+
+    @LazyProperty
+    def changed(self):
+        return self.state is NodeState.CHANGED
+
+    @LazyProperty
+    def not_changed(self):
+        return self.state is NodeState.NOT_CHANGED
+
+    @LazyProperty
+    def removed(self):
+        return self.state is NodeState.REMOVED
+
+
+class FileNode(Node):
+    """
+    Class representing file nodes.
+
+    :attribute: path: path to the node, relative to repostiory's root
+    :attribute: content: if given arbitrary sets content of the file
+    :attribute: changeset: if given, first time content is accessed, callback
+    :attribute: mode: octal stat mode for a node. Default is 0100644.
+    """
+
+    def __init__(self, path, content=None, changeset=None, mode=None):
+        """
+        Only one of ``content`` and ``changeset`` may be given. Passing both
+        would raise ``NodeError`` exception.
+
+        :param path: relative path to the node
+        :param content: content may be passed to constructor
+        :param changeset: if given, will use it to lazily fetch content
+        :param mode: octal representation of ST_MODE (i.e. 0100644)
+        """
+
+        if content and changeset:
+            raise NodeError("Cannot use both content and changeset")
+        super(FileNode, self).__init__(path, kind=NodeKind.FILE)
+        self.changeset = changeset
+        self._content = content
+        self._mode = mode or 0100644
+
+    @LazyProperty
+    def mode(self):
+        """
+        Returns lazily mode of the FileNode. If ``changeset`` is not set, would
+        use value given at initialization or 0100644 (default).
+        """
+        if self.changeset:
+            mode = self.changeset.get_file_mode(self.path)
+        else:
+            mode = self._mode
+        return mode
+
+    @property
+    def content(self):
+        """
+        Returns lazily content of the FileNode. If possible, would try to
+        decode content from UTF-8.
+        """
+        if self.changeset:
+            content = self.changeset.get_file_content(self.path)
+        else:
+            content = self._content
+
+        if bool(content and '\0' in content):
+            return content
+        return safe_unicode(content)
+
+    @LazyProperty
+    def size(self):
+        if self.changeset:
+            return self.changeset.get_file_size(self.path)
+        raise NodeError("Cannot retrieve size of the file without related "
+            "changeset attribute")
+
+    @LazyProperty
+    def message(self):
+        if self.changeset:
+            return self.last_changeset.message
+        raise NodeError("Cannot retrieve message of the file without related "
+            "changeset attribute")
+
+    @LazyProperty
+    def last_changeset(self):
+        if self.changeset:
+            return self.changeset.get_file_changeset(self.path)
+        raise NodeError("Cannot retrieve last changeset of the file without "
+            "related changeset attribute")
+
+    def get_mimetype(self):
+        """
+        Mimetype is calculated based on the file's content. If ``_mimetype``
+        attribute is available, it will be returned (backends which store
+        mimetypes or can easily recognize them, should set this private
+        attribute to indicate that type should *NOT* be calculated).
+        """
+        if hasattr(self, '_mimetype'):
+            if (isinstance(self._mimetype,(tuple,list,)) and
+                len(self._mimetype) == 2):
+                return self._mimetype
+            else:
+                raise NodeError('given _mimetype attribute must be an 2 '
+                               'element list or tuple')
+
+        mtype,encoding = mimetypes.guess_type(self.name)
+
+        if mtype is None:
+            if self.is_binary:
+                mtype = 'application/octet-stream'
+                encoding = None
+            else:
+                mtype = 'text/plain'
+                encoding = None
+        return mtype,encoding
+
+    @LazyProperty
+    def mimetype(self):
+        """
+        Wrapper around full mimetype info. It returns only type of fetched
+        mimetype without the encoding part. use get_mimetype function to fetch
+        full set of (type,encoding)
+        """
+        return self.get_mimetype()[0]
+
+    @LazyProperty
+    def mimetype_main(self):
+        return self.mimetype.split('/')[0]
+
+    @LazyProperty
+    def lexer(self):
+        """
+        Returns pygment's lexer class. Would try to guess lexer taking file's
+        content, name and mimetype.
+        """
+        try:
+            lexer = lexers.guess_lexer_for_filename(self.name, self.content)
+        except lexers.ClassNotFound:
+            lexer = lexers.TextLexer()
+        # returns first alias
+        return lexer
+
+    @LazyProperty
+    def lexer_alias(self):
+        """
+        Returns first alias of the lexer guessed for this file.
+        """
+        return self.lexer.aliases[0]
+
+    @LazyProperty
+    def history(self):
+        """
+        Returns a list of changeset for this file in which the file was changed
+        """
+        if self.changeset is None:
+            raise NodeError('Unable to get changeset for this FileNode')
+        return self.changeset.get_file_history(self.path)
+
+    @LazyProperty
+    def annotate(self):
+        """
+        Returns a list of three element tuples with lineno,changeset and line
+        """
+        if self.changeset is None:
+            raise NodeError('Unable to get changeset for this FileNode')
+        return self.changeset.get_file_annotate(self.path)
+
+    @LazyProperty
+    def state(self):
+        if not self.changeset:
+            raise NodeError("Cannot check state of the node if it's not "
+                "linked with changeset")
+        elif self.path in (node.path for node in self.changeset.added):
+            return NodeState.ADDED
+        elif self.path in (node.path for node in self.changeset.changed):
+            return NodeState.CHANGED
+        else:
+            return NodeState.NOT_CHANGED
+
+    @property
+    def is_binary(self):
+        """
+        Returns True if file has binary content.
+        """
+        bin = '\0' in self.content
+        return bin
+
+    @LazyProperty
+    def extension(self):
+        """Returns filenode extension"""
+        return self.name.split('.')[-1]
+
+    def is_executable(self):
+        """
+        Returns ``True`` if file has executable flag turned on.
+        """
+        return bool(self.mode & stat.S_IXUSR)
+
+
+class RemovedFileNode(FileNode):
+    """
+    Dummy FileNode class - trying to access any public attribute except path,
+    name, kind or state (or methods/attributes checking those two) would raise
+    RemovedFileNodeError.
+    """
+    ALLOWED_ATTRIBUTES = ['name', 'path', 'state', 'is_root', 'is_file',
+        'is_dir', 'kind', 'added', 'changed', 'not_changed', 'removed']
+
+    def __init__(self, path):
+        """
+        :param path: relative path to the node
+        """
+        super(RemovedFileNode, self).__init__(path=path)
+
+    def __getattribute__(self, attr):
+        if attr.startswith('_') or attr in RemovedFileNode.ALLOWED_ATTRIBUTES:
+            return super(RemovedFileNode, self).__getattribute__(attr)
+        raise RemovedFileNodeError("Cannot access attribute %s on "
+            "RemovedFileNode" % attr)
+
+    @LazyProperty
+    def state(self):
+        return NodeState.REMOVED
+
+
+class DirNode(Node):
+    """
+    DirNode stores list of files and directories within this node.
+    Nodes may be used standalone but within repository context they
+    lazily fetch data within same repositorty's changeset.
+    """
+
+    def __init__(self, path, nodes=(), changeset=None):
+        """
+        Only one of ``nodes`` and ``changeset`` may be given. Passing both
+        would raise ``NodeError`` exception.
+
+        :param path: relative path to the node
+        :param nodes: content may be passed to constructor
+        :param changeset: if given, will use it to lazily fetch content
+        :param size: always 0 for ``DirNode``
+        """
+        if nodes and changeset:
+            raise NodeError("Cannot use both nodes and changeset")
+        super(DirNode, self).__init__(path, NodeKind.DIR)
+        self.changeset = changeset
+        self._nodes = nodes
+
+    @LazyProperty
+    def content(self):
+        raise NodeError("%s represents a dir and has no ``content`` attribute"
+            % self)
+
+    @LazyProperty
+    def nodes(self):
+        if self.changeset:
+            nodes = self.changeset.get_nodes(self.path)
+        else:
+            nodes = self._nodes
+        self._nodes_dict = dict((node.path, node) for node in nodes)
+        return sorted(nodes)
+
+    @LazyProperty
+    def files(self):
+        return sorted((node for node in self.nodes if node.is_file()))
+
+    @LazyProperty
+    def dirs(self):
+        return sorted((node for node in self.nodes if node.is_dir()))
+
+    def __iter__(self):
+        for node in self.nodes:
+            yield node
+
+    def get_node(self, path):
+        """
+        Returns node from within this particular ``DirNode``, so it is now
+        allowed to fetch, i.e. node located at 'docs/api/index.rst' from node
+        'docs'. In order to access deeper nodes one must fetch nodes between
+        them first - this would work::
+
+           docs = root.get_node('docs')
+           docs.get_node('api').get_node('index.rst')
+
+        :param: path - relative to the current node
+
+        .. note::
+           To access lazily (as in example above) node have to be initialized
+           with related changeset object - without it node is out of
+           context and may know nothing about anything else than nearest
+           (located at same level) nodes.
+        """
+        try:
+            path = path.rstrip('/')
+            if path == '':
+                raise NodeError("Cannot retrieve node without path")
+            self.nodes  # access nodes first in order to set _nodes_dict
+            paths = path.split('/')
+            if len(paths) == 1:
+                if not self.is_root():
+                    path = '/'.join((self.path, paths[0]))
+                else:
+                    path = paths[0]
+                return self._nodes_dict[path]
+            elif len(paths) > 1:
+                if self.changeset is None:
+                    raise NodeError("Cannot access deeper "
+                                    "nodes without changeset")
+                else:
+                    path1, path2 = paths[0], '/'.join(paths[1:])
+                    return self.get_node(path1).get_node(path2)
+            else:
+                raise KeyError
+        except KeyError:
+            raise NodeError("Node does not exist at %s" % path)
+
+    @LazyProperty
+    def state(self):
+        raise NodeError("Cannot access state of DirNode")
+
+    @LazyProperty
+    def size(self):
+        size = 0
+        for root, dirs, files in self.changeset.walk(self.path):
+            for f in files:
+                size += f.size
+
+        return size
+
+
+class RootNode(DirNode):
+    """
+    DirNode being the root node of the repository.
+    """
+
+    def __init__(self, nodes=(), changeset=None):
+        super(RootNode, self).__init__(path='', nodes=nodes,
+            changeset=changeset)
+
+    def __repr__(self):
+        return '<%s>' % self.__class__.__name__

rhodecode/lib/vcs/utils/__init__.py

@@ -0,0 +1,133 b''
+"""
+This module provides some useful tools for ``vcs`` like annotate/diff html
+output. It also includes some internal helpers.
+"""
+import sys
+import time
+import datetime
+
+
+def makedate():
+    lt = time.localtime()
+    if lt[8] == 1 and time.daylight:
+        tz = time.altzone
+    else:
+        tz = time.timezone
+    return time.mktime(lt), tz
+
+
+def date_fromtimestamp(unixts, tzoffset=0):
+    """
+    Makes a local datetime object out of unix timestamp
+
+    :param unixts:
+    :param tzoffset:
+    """
+
+    return datetime.datetime.fromtimestamp(float(unixts))
+
+
+def safe_unicode(str_, from_encoding='utf8'):
+    """
+    safe unicode function. Does few trick to turn str_ into unicode
+
+    In case of UnicodeDecode error we try to return it with encoding detected
+    by chardet library if it fails fallback to unicode with errors replaced
+
+    :param str_: string to decode
+    :rtype: unicode
+    :returns: unicode object
+    """
+    if isinstance(str_, unicode):
+        return str_
+
+    try:
+        return unicode(str_)
+    except UnicodeDecodeError:
+        pass
+
+    try:
+        return unicode(str_, from_encoding)
+    except UnicodeDecodeError:
+        pass
+
+    try:
+        import chardet
+        encoding = chardet.detect(str_)['encoding']
+        if encoding is None:
+            raise Exception()
+        return str_.decode(encoding)
+    except (ImportError, UnicodeDecodeError, Exception):
+        return unicode(str_, from_encoding, 'replace')
+
+
+def safe_str(unicode_, to_encoding='utf8'):
+    """
+    safe str function. Does few trick to turn unicode_ into string
+
+    In case of UnicodeEncodeError we try to return it with encoding detected
+    by chardet library if it fails fallback to string with errors replaced
+
+    :param unicode_: unicode to encode
+    :rtype: str
+    :returns: str object
+    """
+
+    if isinstance(unicode_, str):
+        return unicode_
+
+    try:
+        return unicode_.encode(to_encoding)
+    except UnicodeEncodeError:
+        pass
+
+    try:
+        import chardet
+        encoding = chardet.detect(unicode_)['encoding']
+        print encoding
+        if encoding is None:
+            raise UnicodeEncodeError()
+
+        return unicode_.encode(encoding)
+    except (ImportError, UnicodeEncodeError):
+        return unicode_.encode(to_encoding, 'replace')
+
+    return safe_str
+
+
+def author_email(author):
+    """
+    returns email address of given author.
+    If any of <,> sign are found, it fallbacks to regex findall()
+    and returns first found result or empty string
+
+    Regex taken from http://www.regular-expressions.info/email.html
+    """
+    import re
+    r = author.find('>')
+    l = author.find('<')
+
+    if l == -1 or r == -1:
+        # fallback to regex match of email out of a string
+        email_re = re.compile(r"""[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!"""
+                              r"""#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z"""
+                              r"""0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]"""
+                              r"""*[a-z0-9])?""", re.IGNORECASE)
+        m = re.findall(email_re, author)
+        return m[0] if m else ''
+
+    return author[l + 1:r].strip()
+
+
+def author_name(author):
+    """
+    get name of author, or else username.
+    It'll try to find an email in the author string and just cut it off
+    to get the username
+    """
+
+    if not '@' in author:
+        return author
+    else:
+        return author.replace(author_email(author), '').replace('<', '')\
+            .replace('>', '').strip()

rhodecode/lib/vcs/utils/annotate.py

@@ -0,0 +1,177 b''
+from rhodecode.lib.vcs.exceptions import VCSError
+from rhodecode.lib.vcs.nodes import FileNode
+from pygments.formatters import HtmlFormatter
+from pygments import highlight
+
+import StringIO
+
+
+def annotate_highlight(filenode, annotate_from_changeset_func=None,
+        order=None, headers=None, **options):
+    """
+    Returns html portion containing annotated table with 3 columns: line
+    numbers, changeset information and pygmentized line of code.
+
+    :param filenode: FileNode object
+    :param annotate_from_changeset_func: function taking changeset and
+      returning single annotate cell; needs break line at the end
+    :param order: ordered sequence of ``ls`` (line numbers column),
+      ``annotate`` (annotate column), ``code`` (code column); Default is
+      ``['ls', 'annotate', 'code']``
+    :param headers: dictionary with headers (keys are whats in ``order``
+      parameter)
+    """
+    options['linenos'] = True
+    formatter = AnnotateHtmlFormatter(filenode=filenode, order=order,
+        headers=headers,
+        annotate_from_changeset_func=annotate_from_changeset_func, **options)
+    lexer = filenode.lexer
+    highlighted = highlight(filenode.content, lexer, formatter)
+    return highlighted
+
+
+class AnnotateHtmlFormatter(HtmlFormatter):
+
+    def __init__(self, filenode, annotate_from_changeset_func=None,
+            order=None, **options):
+        """
+        If ``annotate_from_changeset_func`` is passed it should be a function
+        which returns string from the given changeset. For example, we may pass
+        following function as ``annotate_from_changeset_func``::
+
+            def changeset_to_anchor(changeset):
+                return '<a href="/changesets/%s/">%s</a>\n' %\
+                       (changeset.id, changeset.id)
+
+        :param annotate_from_changeset_func: see above
+        :param order: (default: ``['ls', 'annotate', 'code']``); order of
+          columns;
+        :param options: standard pygment's HtmlFormatter options, there is
+          extra option tough, ``headers``. For instance we can pass::
+
+             formatter = AnnotateHtmlFormatter(filenode, headers={
+                'ls': '#',
+                'annotate': 'Annotate',
+                'code': 'Code',
+             })
+
+        """
+        super(AnnotateHtmlFormatter, self).__init__(**options)
+        self.annotate_from_changeset_func = annotate_from_changeset_func
+        self.order = order or ('ls', 'annotate', 'code')
+        headers = options.pop('headers', None)
+        if headers and not ('ls' in headers and 'annotate' in headers and
+            'code' in headers):
+            raise ValueError("If headers option dict is specified it must "
+                "all 'ls', 'annotate' and 'code' keys")
+        self.headers = headers
+        if isinstance(filenode, FileNode):
+            self.filenode = filenode
+        else:
+            raise VCSError("This formatter expect FileNode parameter, not %r"
+                % type(filenode))
+
+    def annotate_from_changeset(self, changeset):
+        """
+        Returns full html line for single changeset per annotated line.
+        """
+        if self.annotate_from_changeset_func:
+            return self.annotate_from_changeset_func(changeset)
+        else:
+            return ''.join((changeset.id, '\n'))
+
+    def _wrap_tablelinenos(self, inner):
+        dummyoutfile = StringIO.StringIO()
+        lncount = 0
+        for t, line in inner:
+            if t:
+                lncount += 1
+            dummyoutfile.write(line)
+
+        fl = self.linenostart
+        mw = len(str(lncount + fl - 1))
+        sp = self.linenospecial
+        st = self.linenostep
+        la = self.lineanchors
+        aln = self.anchorlinenos
+        if sp:
+            lines = []
+
+            for i in range(fl, fl + lncount):
+                if i % st == 0:
+                    if i % sp == 0:
+                        if aln:
+                            lines.append('<a href="#%s-%d" class="special">'
+                                         '%*d</a>' %
+                                         (la, i, mw, i))
+                        else:
+                            lines.append('<span class="special">'
+                                         '%*d</span>' % (mw, i))
+                    else:
+                        if aln:
+                            lines.append('<a href="#%s-%d">'
+                                         '%*d</a>' % (la, i, mw, i))
+                        else:
+                            lines.append('%*d' % (mw, i))
+                else:
+                    lines.append('')
+            ls = '\n'.join(lines)
+        else:
+            lines = []
+            for i in range(fl, fl + lncount):
+                if i % st == 0:
+                    if aln:
+                        lines.append('<a href="#%s-%d">%*d</a>' \
+                                     % (la, i, mw, i))
+                    else:
+                        lines.append('%*d' % (mw, i))
+                else:
+                    lines.append('')
+            ls = '\n'.join(lines)
+
+        annotate_changesets = [tup[1] for tup in self.filenode.annotate]
+        # If pygments cropped last lines break we need do that too
+        ln_cs = len(annotate_changesets)
+        ln_ = len(ls.splitlines())
+        if  ln_cs > ln_:
+            annotate_changesets = annotate_changesets[:ln_ - ln_cs]
+        annotate = ''.join((self.annotate_from_changeset(changeset)
+            for changeset in annotate_changesets))
+        # in case you wonder about the seemingly redundant <div> here:
+        # since the content in the other cell also is wrapped in a div,
+        # some browsers in some configurations seem to mess up the formatting.
+        '''
+        yield 0, ('<table class="%stable">' % self.cssclass +
+                  '<tr><td class="linenos"><div class="linenodiv"><pre>' +
+                  ls + '</pre></div></td>' +
+                  '<td class="code">')
+        yield 0, dummyoutfile.getvalue()
+        yield 0, '</td></tr></table>'
+
+        '''
+        headers_row = []
+        if self.headers:
+            headers_row = ['<tr class="annotate-header">']
+            for key in self.order:
+                td = ''.join(('<td>', self.headers[key], '</td>'))
+                headers_row.append(td)
+            headers_row.append('</tr>')
+
+        body_row_start = ['<tr>']
+        for key in self.order:
+            if key == 'ls':
+                body_row_start.append(
+                    '<td class="linenos"><div class="linenodiv"><pre>' +
+                    ls + '</pre></div></td>')
+            elif key == 'annotate':
+                body_row_start.append(
+                    '<td class="annotate"><div class="annotatediv"><pre>' +
+                    annotate + '</pre></div></td>')
+            elif key == 'code':
+                body_row_start.append('<td class="code">')
+        yield 0, ('<table class="%stable">' % self.cssclass +
+                  ''.join(headers_row) +
+                  ''.join(body_row_start)
+                  )
+        yield 0, dummyoutfile.getvalue()
+        yield 0, '</td></tr></table>'

rhodecode/lib/vcs/utils/archivers.py

@@ -0,0 +1,67 b''
+# -*- coding: utf-8 -*-
+"""
+    vcs.utils.archivers
+    ~~~~~~~~~~~~~~~~~~~
+
+    set of archiver functions for creating archives from repository content
+
+    :created_on: Jan 21, 2011
+    :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
+"""
+
+
+class BaseArchiver(object):
+
+    def __init__(self):
+        self.archive_file = self._get_archive_file()
+
+    def addfile(self):
+        """
+        Adds a file to archive container
+        """
+        pass
+
+    def close(self):
+        """
+        Closes and finalizes operation of archive container object
+        """
+        self.archive_file.close()
+
+    def _get_archive_file(self):
+        """
+        Returns container for specific archive
+        """
+        raise NotImplementedError()
+
+
+class TarArchiver(BaseArchiver):
+    pass
+
+
+class Tbz2Archiver(BaseArchiver):
+    pass
+
+
+class TgzArchiver(BaseArchiver):
+    pass
+
+
+class ZipArchiver(BaseArchiver):
+    pass
+
+
+def get_archiver(self, kind):
+    """
+    Returns instance of archiver class specific to given kind
+
+    :param kind: archive kind
+    """
+
+    archivers = {
+        'tar': TarArchiver,
+        'tbz2': Tbz2Archiver,
+        'tgz': TgzArchiver,
+        'zip': ZipArchiver,
+    }
+
+    return archivers[kind]()

rhodecode/lib/vcs/utils/baseui_config.py

@@ -0,0 +1,47 b''
+from mercurial import ui, config
+
+
+def make_ui(self, path='hgwebdir.config'):
+    """
+    A funcion that will read python rc files and make an ui from read options
+
+    :param path: path to mercurial config file
+    """
+    #propagated from mercurial documentation
+    sections = [
+                'alias',
+                'auth',
+                'decode/encode',
+                'defaults',
+                'diff',
+                'email',
+                'extensions',
+                'format',
+                'merge-patterns',
+                'merge-tools',
+                'hooks',
+                'http_proxy',
+                'smtp',
+                'patch',
+                'paths',
+                'profiling',
+                'server',
+                'trusted',
+                'ui',
+                'web',
+                ]
+
+    repos = path
+    baseui = ui.ui()
+    cfg = config.config()
+    cfg.read(repos)
+    self.paths = cfg.items('paths')
+    self.base_path = self.paths[0][1].replace('*', '')
+    self.check_repo_dir(self.paths)
+    self.set_statics(cfg)
+
+    for section in sections:
+        for k, v in cfg.items(section):
+            baseui.setconfig(section, k, v)
+
+    return baseui

rhodecode/lib/vcs/utils/compat.py

@@ -0,0 +1,13 b''
+"""
+Various utilities to work with Python < 2.7.
+
+Those utilities may be deleted once ``vcs`` stops support for older Python
+versions.
+"""
+import sys
+
+
+if sys.version_info >= (2, 7):
+    unittest = __import__('unittest')
+else:
+    unittest = __import__('unittest2')

rhodecode/lib/vcs/utils/diffs.py

@@ -0,0 +1,460 b''
+# -*- coding: utf-8 -*-
+# original copyright: 2007-2008 by Armin Ronacher
+# licensed under the BSD license.
+
+import re
+import difflib
+import logging
+
+from difflib import unified_diff
+from itertools import tee, imap
+
+from mercurial.match import match
+
+from rhodecode.lib.vcs.exceptions import VCSError
+from rhodecode.lib.vcs.nodes import FileNode, NodeError
+
+
+def get_udiff(filenode_old, filenode_new,show_whitespace=True):
+    """
+    Returns unified diff between given ``filenode_old`` and ``filenode_new``.
+    """
+    try:
+        filenode_old_date = filenode_old.last_changeset.date
+    except NodeError:
+        filenode_old_date = None
+
+    try:
+        filenode_new_date = filenode_new.last_changeset.date
+    except NodeError:
+        filenode_new_date = None
+
+    for filenode in (filenode_old, filenode_new):
+        if not isinstance(filenode, FileNode):
+            raise VCSError("Given object should be FileNode object, not %s"
+                % filenode.__class__)
+
+    if filenode_old_date and filenode_new_date:
+        if not filenode_old_date < filenode_new_date:
+            logging.debug("Generating udiff for filenodes with not increasing "
+                "dates")
+
+    vcs_udiff = unified_diff(filenode_old.content.splitlines(True),
+                               filenode_new.content.splitlines(True),
+                               filenode_old.name,
+                               filenode_new.name,
+                               filenode_old_date,
+                               filenode_old_date)
+    return vcs_udiff
+
+
+def get_gitdiff(filenode_old, filenode_new, ignore_whitespace=True):
+    """
+    Returns git style diff between given ``filenode_old`` and ``filenode_new``.
+
+    :param ignore_whitespace: ignore whitespaces in diff
+    """
+
+    for filenode in (filenode_old, filenode_new):
+        if not isinstance(filenode, FileNode):
+            raise VCSError("Given object should be FileNode object, not %s"
+                % filenode.__class__)
+
+    old_raw_id = getattr(filenode_old.changeset, 'raw_id', '0' * 40)
+    new_raw_id = getattr(filenode_new.changeset, 'raw_id', '0' * 40)
+
+    repo = filenode_new.changeset.repository
+    vcs_gitdiff = repo._get_diff(old_raw_id, new_raw_id, filenode_new.path,
+                                 ignore_whitespace)
+
+    return vcs_gitdiff
+
+
+class DiffProcessor(object):
+    """
+    Give it a unified diff and it returns a list of the files that were
+    mentioned in the diff together with a dict of meta information that
+    can be used to render it in a HTML template.
+    """
+    _chunk_re = re.compile(r'@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)')
+
+    def __init__(self, diff, differ='diff', format='udiff'):
+        """
+        :param diff:   a text in diff format or generator
+        :param format: format of diff passed, `udiff` or `gitdiff`
+        """
+        if isinstance(diff, basestring):
+            diff = [diff]
+
+        self.__udiff = diff
+        self.__format = format
+        self.adds = 0
+        self.removes = 0
+
+        if isinstance(self.__udiff, basestring):
+            self.lines = iter(self.__udiff.splitlines(1))
+
+        elif self.__format == 'gitdiff':
+            udiff_copy = self.copy_iterator()
+            self.lines = imap(self.escaper, self._parse_gitdiff(udiff_copy))
+        else:
+            udiff_copy = self.copy_iterator()
+            self.lines = imap(self.escaper, udiff_copy)
+
+        # Select a differ.
+        if differ == 'difflib':
+            self.differ = self._highlight_line_difflib
+        else:
+            self.differ = self._highlight_line_udiff
+
+    def escaper(self, string):
+        return string.replace('<', '&lt;').replace('>', '&gt;')
+
+    def copy_iterator(self):
+        """
+        make a fresh copy of generator, we should not iterate thru
+        an original as it's needed for repeating operations on
+        this instance of DiffProcessor
+        """
+        self.__udiff, iterator_copy = tee(self.__udiff)
+        return iterator_copy
+
+    def _extract_rev(self, line1, line2):
+        """
+        Extract the filename and revision hint from a line.
+        """
+
+        try:
+            if line1.startswith('--- ') and line2.startswith('+++ '):
+                l1 = line1[4:].split(None, 1)
+                old_filename = l1[0].lstrip('a/') if len(l1) >= 1 else None
+                old_rev = l1[1] if len(l1) == 2 else 'old'
+
+                l2 = line2[4:].split(None, 1)
+                new_filename = l2[0].lstrip('b/') if len(l1) >= 1 else None
+                new_rev = l2[1] if len(l2) == 2 else 'new'
+
+                filename = old_filename if (old_filename !=
+                                            'dev/null') else new_filename
+
+                return filename, new_rev, old_rev
+        except (ValueError, IndexError):
+            pass
+
+        return None, None, None
+
+    def _parse_gitdiff(self, diffiterator):
+        def line_decoder(l):
+            if l.startswith('+') and not l.startswith('+++'):
+                self.adds += 1
+            elif l.startswith('-') and not l.startswith('---'):
+                self.removes += 1
+            return l.decode('utf8', 'replace')
+
+        output = list(diffiterator)
+        size = len(output)
+
+        if size == 2:
+            l = []
+            l.extend([output[0]])
+            l.extend(output[1].splitlines(1))
+            return map(line_decoder, l)
+        elif size == 1:
+            return  map(line_decoder, output[0].splitlines(1))
+        elif size == 0:
+            return []
+
+        raise Exception('wrong size of diff %s' % size)
+
+    def _highlight_line_difflib(self, line, next):
+        """
+        Highlight inline changes in both lines.
+        """
+
+        if line['action'] == 'del':
+            old, new = line, next
+        else:
+            old, new = next, line
+
+        oldwords = re.split(r'(\W)', old['line'])
+        newwords = re.split(r'(\W)', new['line'])
+
+        sequence = difflib.SequenceMatcher(None, oldwords, newwords)
+
+        oldfragments, newfragments = [], []
+        for tag, i1, i2, j1, j2 in sequence.get_opcodes():
+            oldfrag = ''.join(oldwords[i1:i2])
+            newfrag = ''.join(newwords[j1:j2])
+            if tag != 'equal':
+                if oldfrag:
+                    oldfrag = '<del>%s</del>' % oldfrag
+                if newfrag:
+                    newfrag = '<ins>%s</ins>' % newfrag
+            oldfragments.append(oldfrag)
+            newfragments.append(newfrag)
+
+        old['line'] = "".join(oldfragments)
+        new['line'] = "".join(newfragments)
+
+    def _highlight_line_udiff(self, line, next):
+        """
+        Highlight inline changes in both lines.
+        """
+        start = 0
+        limit = min(len(line['line']), len(next['line']))
+        while start < limit and line['line'][start] == next['line'][start]:
+            start += 1
+        end = -1
+        limit -= start
+        while -end <= limit and line['line'][end] == next['line'][end]:
+            end -= 1
+        end += 1
+        if start or end:
+            def do(l):
+                last = end + len(l['line'])
+                if l['action'] == 'add':
+                    tag = 'ins'
+                else:
+                    tag = 'del'
+                l['line'] = '%s<%s>%s</%s>%s' % (
+                    l['line'][:start],
+                    tag,
+                    l['line'][start:last],
+                    tag,
+                    l['line'][last:]
+                )
+            do(line)
+            do(next)
+
+    def _parse_udiff(self):
+        """
+        Parse the diff an return data for the template.
+        """
+        lineiter = self.lines
+        files = []
+        try:
+            line = lineiter.next()
+            # skip first context
+            skipfirst = True
+            while 1:
+                # continue until we found the old file
+                if not line.startswith('--- '):
+                    line = lineiter.next()
+                    continue
+
+                chunks = []
+                filename, old_rev, new_rev = \
+                    self._extract_rev(line, lineiter.next())
+                files.append({
+                    'filename':         filename,
+                    'old_revision':     old_rev,
+                    'new_revision':     new_rev,
+                    'chunks':           chunks
+                })
+
+                line = lineiter.next()
+                while line:
+                    match = self._chunk_re.match(line)
+                    if not match:
+                        break
+
+                    lines = []
+                    chunks.append(lines)
+
+                    old_line, old_end, new_line, new_end = \
+                        [int(x or 1) for x in match.groups()[:-1]]
+                    old_line -= 1
+                    new_line -= 1
+                    context = len(match.groups()) == 5
+                    old_end += old_line
+                    new_end += new_line
+
+                    if context:
+                        if not skipfirst:
+                            lines.append({
+                                'old_lineno': '...',
+                                'new_lineno': '...',
+                                'action': 'context',
+                                'line': line,
+                            })
+                        else:
+                            skipfirst = False
+
+                    line = lineiter.next()
+                    while old_line < old_end or new_line < new_end:
+                        if line:
+                            command, line = line[0], line[1:]
+                        else:
+                            command = ' '
+                        affects_old = affects_new = False
+
+                        # ignore those if we don't expect them
+                        if command in '#@':
+                            continue
+                        elif command == '+':
+                            affects_new = True
+                            action = 'add'
+                        elif command == '-':
+                            affects_old = True
+                            action = 'del'
+                        else:
+                            affects_old = affects_new = True
+                            action = 'unmod'
+
+                        old_line += affects_old
+                        new_line += affects_new
+                        lines.append({
+                            'old_lineno':   affects_old and old_line or '',
+                            'new_lineno':   affects_new and new_line or '',
+                            'action':       action,
+                            'line':         line
+                        })
+                        line = lineiter.next()
+
+        except StopIteration:
+            pass
+
+        # highlight inline changes
+        for file in files:
+            for chunk in chunks:
+                lineiter = iter(chunk)
+                #first = True
+                try:
+                    while 1:
+                        line = lineiter.next()
+                        if line['action'] != 'unmod':
+                            nextline = lineiter.next()
+                            if nextline['action'] == 'unmod' or \
+                               nextline['action'] == line['action']:
+                                continue
+                            self.differ(line, nextline)
+                except StopIteration:
+                    pass
+
+        return files
+
+    def prepare(self):
+        """
+        Prepare the passed udiff for HTML rendering. It'l return a list
+        of dicts
+        """
+        return self._parse_udiff()
+
+    def _safe_id(self, idstring):
+        """Make a string safe for including in an id attribute.
+
+        The HTML spec says that id attributes 'must begin with
+        a letter ([A-Za-z]) and may be followed by any number
+        of letters, digits ([0-9]), hyphens ("-"), underscores
+        ("_"), colons (":"), and periods (".")'. These regexps
+        are slightly over-zealous, in that they remove colons
+        and periods unnecessarily.
+
+        Whitespace is transformed into underscores, and then
+        anything which is not a hyphen or a character that
+        matches \w (alphanumerics and underscore) is removed.
+
+        """
+        # Transform all whitespace to underscore
+        idstring = re.sub(r'\s', "_", '%s' % idstring)
+        # Remove everything that is not a hyphen or a member of \w
+        idstring = re.sub(r'(?!-)\W', "", idstring).lower()
+        return idstring
+
+    def raw_diff(self):
+        """
+        Returns raw string as udiff
+        """
+        udiff_copy = self.copy_iterator()
+        if self.__format == 'gitdiff':
+            udiff_copy = self._parse_gitdiff(udiff_copy)
+        return u''.join(udiff_copy)
+
+    def as_html(self, table_class='code-difftable', line_class='line',
+                new_lineno_class='lineno old', old_lineno_class='lineno new',
+                code_class='code'):
+        """
+        Return udiff as html table with customized css classes
+        """
+        def _link_to_if(condition, label, url):
+            """
+            Generates a link if condition is meet or just the label if not.
+            """
+
+            if condition:
+                return '''<a href="%(url)s">%(label)s</a>''' % {'url': url,
+                                                                'label': label}
+            else:
+                return label
+        diff_lines = self.prepare()
+        _html_empty = True
+        _html = []
+        _html.append('''<table class="%(table_class)s">\n''' \
+                                            % {'table_class': table_class})
+        for diff in diff_lines:
+            for line in diff['chunks']:
+                _html_empty = False
+                for change in line:
+                    _html.append('''<tr class="%(line_class)s %(action)s">\n''' \
+                        % {'line_class': line_class,
+                           'action': change['action']})
+                    anchor_old_id = ''
+                    anchor_new_id = ''
+                    anchor_old = "%(filename)s_o%(oldline_no)s" % \
+                                {'filename': self._safe_id(diff['filename']),
+                                 'oldline_no': change['old_lineno']}
+                    anchor_new = "%(filename)s_n%(oldline_no)s" % \
+                                {'filename': self._safe_id(diff['filename']),
+                                 'oldline_no': change['new_lineno']}
+                    cond_old = change['old_lineno'] != '...' and \
+                                                        change['old_lineno']
+                    cond_new = change['new_lineno'] != '...' and \
+                                                        change['new_lineno']
+                    if cond_old:
+                        anchor_old_id = 'id="%s"' % anchor_old
+                    if cond_new:
+                        anchor_new_id = 'id="%s"' % anchor_new
+                    ###########################################################
+                    # OLD LINE NUMBER
+                    ###########################################################
+                    _html.append('''\t<td %(a_id)s class="%(old_lineno_cls)s">''' \
+                                    % {'a_id': anchor_old_id,
+                                       'old_lineno_cls': old_lineno_class})
+
+                    _html.append('''<pre>%(link)s</pre>''' \
+                        % {'link':
+                        _link_to_if(cond_old, change['old_lineno'], '#%s' \
+                                                                % anchor_old)})
+                    _html.append('''</td>\n''')
+                    ###########################################################
+                    # NEW LINE NUMBER
+                    ###########################################################
+
+                    _html.append('''\t<td %(a_id)s class="%(new_lineno_cls)s">''' \
+                                    % {'a_id': anchor_new_id,
+                                       'new_lineno_cls': new_lineno_class})
+
+                    _html.append('''<pre>%(link)s</pre>''' \
+                        % {'link':
+                        _link_to_if(cond_new, change['new_lineno'], '#%s' \
+                                                                % anchor_new)})
+                    _html.append('''</td>\n''')
+                    ###########################################################
+                    # CODE
+                    ###########################################################
+                    _html.append('''\t<td class="%(code_class)s">''' \
+                                                % {'code_class': code_class})
+                    _html.append('''\n\t\t<pre>%(code)s</pre>\n''' \
+                                                % {'code': change['line']})
+                    _html.append('''\t</td>''')
+                    _html.append('''\n</tr>\n''')
+        _html.append('''</table>''')
+        if _html_empty:
+            return None
+        return ''.join(_html)
+
+    def stat(self):
+        """
+        Returns tuple of adde,and removed lines for this instance
+        """
+        return self.adds, self.removes

rhodecode/lib/vcs/utils/fakemod.py

@@ -0,0 +1,13 b''
+import imp
+
+
+def create_module(name, path):
+    """
+    Returns module created *on the fly*. Returned module would have name same
+    as given ``name`` and would contain code read from file at the given
+    ``path`` (it may also be a zip or package containing *__main__* module).
+    """
+    module = imp.new_module(name)
+    module.__file__ = path
+    execfile(path, module.__dict__)
+    return module

rhodecode/lib/vcs/utils/filesize.py

@@ -0,0 +1,28 b''
+def filesizeformat(bytes, sep=' '):
+    """
+    Formats the value like a 'human-readable' file size (i.e. 13 KB, 4.1 MB,
+    102 B, 2.3 GB etc).
+
+    Grabbed from Django (http://www.djangoproject.com), slightly modified.
+
+    :param bytes: size in bytes (as integer)
+    :param sep: string separator between number and abbreviation
+    """
+    try:
+        bytes = float(bytes)
+    except (TypeError, ValueError, UnicodeDecodeError):
+        return '0%sB' % sep
+
+    if bytes < 1024:
+        size = bytes
+        template = '%.0f%sB'
+    elif bytes < 1024 * 1024:
+        size = bytes / 1024
+        template = '%.0f%sKB'
+    elif bytes < 1024 * 1024 * 1024:
+        size = bytes / 1024 / 1024
+        template = '%.1f%sMB'
+    else:
+        size = bytes / 1024 / 1024 / 1024
+        template = '%.2f%sGB'
+    return template % (size, sep)

rhodecode/lib/vcs/utils/helpers.py

@@ -0,0 +1,252 b''
+"""
+Utitlites aimed to help achieve mostly basic tasks.
+"""
+from __future__ import division
+
+import re
+import time
+import datetime
+import os.path
+from subprocess import Popen, PIPE
+from rhodecode.lib.vcs.exceptions import VCSError
+from rhodecode.lib.vcs.exceptions import RepositoryError
+from rhodecode.lib.vcs.utils.paths import abspath
+
+ALIASES = ['hg', 'git']
+
+
+def get_scm(path, search_recursively=False, explicit_alias=None):
+    """
+    Returns one of alias from ``ALIASES`` (in order of precedence same as
+    shortcuts given in ``ALIASES``) and top working dir path for the given
+    argument. If no scm-specific directory is found or more than one scm is
+    found at that directory, ``VCSError`` is raised.
+
+    :param search_recursively: if set to ``True``, this function would try to
+      move up to parent directory every time no scm is recognized for the
+      currently checked path. Default: ``False``.
+    :param explicit_alias: can be one of available backend aliases, when given
+      it will return given explicit alias in repositories under more than one
+      version control, if explicit_alias is different than found it will raise
+      VCSError
+    """
+    if not os.path.isdir(path):
+        raise VCSError("Given path %s is not a directory" % path)
+
+    def get_scms(path):
+        return [(scm, path) for scm in get_scms_for_path(path)]
+
+    found_scms = get_scms(path)
+    while  not found_scms and search_recursively:
+        newpath = abspath(path, '..')
+        if newpath == path:
+            break
+        path = newpath
+        found_scms = get_scms(path)
+
+    if len(found_scms) > 1:
+        for scm in found_scms:
+            if scm[0] == explicit_alias:
+                return scm
+        raise VCSError('More than one [%s] scm found at given path %s'
+                       % (','.join((x[0] for x in found_scms)), path))
+
+    if len(found_scms) is 0:
+        raise VCSError('No scm found at given path %s' % path)
+
+    return found_scms[0]
+
+
+def get_scms_for_path(path):
+    """
+    Returns all scm's found at the given path. If no scm is recognized
+    - empty list is returned.
+
+    :param path: path to directory which should be checked. May be callable.
+
+    :raises VCSError: if given ``path`` is not a directory
+    """
+    from rhodecode.lib.vcs.backends import get_backend
+    if hasattr(path, '__call__'):
+        path = path()
+    if not os.path.isdir(path):
+        raise VCSError("Given path %r is not a directory" % path)
+
+    result = []
+    for key in ALIASES:
+        dirname = os.path.join(path, '.' + key)
+        if os.path.isdir(dirname):
+            result.append(key)
+            continue
+        # We still need to check if it's not bare repository as
+        # bare repos don't have working directories
+        try:
+            get_backend(key)(path)
+            result.append(key)
+            continue
+        except RepositoryError:
+            # Wrong backend
+            pass
+        except VCSError:
+            # No backend at all
+            pass
+    return result
+
+
+def get_repo_paths(path):
+    """
+    Returns path's subdirectories which seems to be a repository.
+    """
+    repo_paths = []
+    dirnames = (os.path.abspath(dirname) for dirname in os.listdir(path))
+    for dirname in dirnames:
+        try:
+            get_scm(dirname)
+            repo_paths.append(dirname)
+        except VCSError:
+            pass
+    return repo_paths
+
+
+def run_command(cmd, *args):
+    """
+    Runs command on the system with given ``args``.
+    """
+    command = ' '.join((cmd, args))
+    p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE)
+    stdout, stderr = p.communicate()
+    return p.retcode, stdout, stderr
+
+
+def get_highlighted_code(name, code, type='terminal'):
+    """
+    If pygments are available on the system
+    then returned output is colored. Otherwise
+    unchanged content is returned.
+    """
+    import logging
+    try:
+        import pygments
+        pygments
+    except ImportError:
+        return code
+    from pygments import highlight
+    from pygments.lexers import guess_lexer_for_filename, ClassNotFound
+    from pygments.formatters import TerminalFormatter
+
+    try:
+        lexer = guess_lexer_for_filename(name, code)
+        formatter = TerminalFormatter()
+        content = highlight(code, lexer, formatter)
+    except ClassNotFound:
+        logging.debug("Couldn't guess Lexer, will not use pygments.")
+        content = code
+    return content
+
+def parse_changesets(text):
+    """
+    Returns dictionary with *start*, *main* and *end* ids.
+
+    Examples::
+
+        >>> parse_changesets('aaabbb')
+        {'start': None, 'main': 'aaabbb', 'end': None}
+        >>> parse_changesets('aaabbb..cccddd')
+        {'start': 'aaabbb', 'main': None, 'end': 'cccddd'}
+
+    """
+    text = text.strip()
+    CID_RE = r'[a-zA-Z0-9]+'
+    if not '..' in text:
+        m = re.match(r'^(?P<cid>%s)$' % CID_RE, text)
+        if m:
+            return {
+                'start': None,
+                'main': text,
+                'end': None,
+            }
+    else:
+        RE = r'^(?P<start>%s)?\.{2,3}(?P<end>%s)?$' % (CID_RE, CID_RE)
+        m = re.match(RE, text)
+        if m:
+            result = m.groupdict()
+            result['main'] = None
+            return result
+    raise ValueError("IDs not recognized")
+
+def parse_datetime(text):
+    """
+    Parses given text and returns ``datetime.datetime`` instance or raises
+    ``ValueError``.
+
+    :param text: string of desired date/datetime or something more verbose,
+      like *yesterday*, *2weeks 3days*, etc.
+    """
+
+    text = text.strip().lower()
+
+    INPUT_FORMATS = (
+        '%Y-%m-%d %H:%M:%S',
+        '%Y-%m-%d %H:%M',
+        '%Y-%m-%d',
+        '%m/%d/%Y %H:%M:%S',
+        '%m/%d/%Y %H:%M',
+        '%m/%d/%Y',
+        '%m/%d/%y %H:%M:%S',
+        '%m/%d/%y %H:%M',
+        '%m/%d/%y',
+    )
+    for format in INPUT_FORMATS:
+        try:
+            return datetime.datetime(*time.strptime(text, format)[:6])
+        except ValueError:
+            pass
+
+    # Try descriptive texts
+    if text == 'tomorrow':
+        future = datetime.datetime.now() + datetime.timedelta(days=1)
+        args = future.timetuple()[:3] + (23, 59, 59)
+        return datetime.datetime(*args)
+    elif text == 'today':
+        return datetime.datetime(*datetime.datetime.today().timetuple()[:3])
+    elif text == 'now':
+        return datetime.datetime.now()
+    elif text == 'yesterday':
+        past = datetime.datetime.now() - datetime.timedelta(days=1)
+        return datetime.datetime(*past.timetuple()[:3])
+    else:
+        days = 0
+        matched = re.match(
+            r'^((?P<weeks>\d+) ?w(eeks?)?)? ?((?P<days>\d+) ?d(ays?)?)?$', text)
+        if matched:
+            groupdict = matched.groupdict()
+            if groupdict['days']:
+                days += int(matched.groupdict()['days'])
+            if groupdict['weeks']:
+                days += int(matched.groupdict()['weeks']) * 7
+            past = datetime.datetime.now() - datetime.timedelta(days=days)
+            return datetime.datetime(*past.timetuple()[:3])
+
+    raise ValueError('Wrong date: "%s"' % text)
+
+
+def get_dict_for_attrs(obj, attrs):
+    """
+    Returns dictionary for each attribute from given ``obj``.
+    """
+    data = {}
+    for attr in attrs:
+        data[attr] = getattr(obj, attr)
+    return data
+
+
+def get_total_seconds(timedelta):
+    """
+    Backported for Python 2.5.
+
+    See http://docs.python.org/library/datetime.html.
+    """
+    return ((timedelta.microseconds + (
+            timedelta.seconds +
+            timedelta.days * 24 * 60 * 60
+        ) * 10**6) / 10**6)

rhodecode/lib/vcs/utils/hgcompat.py

@@ -0,0 +1,12 b''
+"""Mercurial libs compatibility
+
+"""
+from mercurial import archival, merge as hg_merge, patch, ui
+from mercurial.commands import clone, nullid, pull
+from mercurial.context import memctx, memfilectx
+from mercurial.error import RepoError, RepoLookupError, Abort
+from mercurial.hgweb.common import get_contact
+from mercurial.localrepo import localrepository
+from mercurial.match import match
+from mercurial.mdiff import diffopts
+from mercurial.node import hex

rhodecode/lib/vcs/utils/imports.py

@@ -0,0 +1,27 b''
+from rhodecode.lib.vcs.exceptions import VCSError
+
+
+def import_class(class_path):
+    """
+    Returns class from the given path.
+
+    For example, in order to get class located at
+    ``vcs.backends.hg.MercurialRepository``:
+
+        try:
+            hgrepo = import_class('vcs.backends.hg.MercurialRepository')
+        except VCSError:
+            # hadle error
+    """
+    splitted = class_path.split('.')
+    mod_path = '.'.join(splitted[:-1])
+    class_name = splitted[-1]
+    try:
+        class_mod = __import__(mod_path, {}, {}, [class_name])
+    except ImportError, err:
+        msg = "There was problem while trying to import backend class. "\
+            "Original error was:\n%s" % err
+        raise VCSError(msg)
+    cls = getattr(class_mod, class_name)
+
+    return cls

rhodecode/lib/vcs/utils/lazy.py

@@ -0,0 +1,27 b''
+class LazyProperty(object):
+    """
+    Decorator for easier creation of ``property`` from potentially expensive to
+    calculate attribute of the class.
+
+    Usage::
+
+      class Foo(object):
+          @LazyProperty
+          def bar(self):
+              print 'Calculating self._bar'
+              return 42
+
+    Taken from http://blog.pythonisito.com/2008/08/lazy-descriptors.html and
+    used widely.
+    """
+
+    def __init__(self, func):
+        self._func = func
+        self.__name__ = func.__name__
+        self.__doc__ = func.__doc__
+
+    def __get__(self, obj, klass=None):
+        if obj is None:
+            return None
+        result = obj.__dict__[self.__name__] = self._func(obj)
+        return result

rhodecode/lib/vcs/utils/lockfiles.py

@@ -0,0 +1,72 b''
+import os
+
+
+class LockFile(object):
+	"""Provides methods to obtain, check for, and release a file based lock which
+	should be used to handle concurrent access to the same file.
+
+	As we are a utility class to be derived from, we only use protected methods.
+
+	Locks will automatically be released on destruction"""
+	__slots__ = ("_file_path", "_owns_lock")
+
+	def __init__(self, file_path):
+		self._file_path = file_path
+		self._owns_lock = False
+
+	def __del__(self):
+		self._release_lock()
+
+	def _lock_file_path(self):
+		""":return: Path to lockfile"""
+		return "%s.lock" % (self._file_path)
+
+	def _has_lock(self):
+		""":return: True if we have a lock and if the lockfile still exists
+		:raise AssertionError: if our lock-file does not exist"""
+		if not self._owns_lock:
+			return False
+
+		return True
+
+	def _obtain_lock_or_raise(self):
+		"""Create a lock file as flag for other instances, mark our instance as lock-holder
+
+		:raise IOError: if a lock was already present or a lock file could not be written"""
+		if self._has_lock():
+			return
+		lock_file = self._lock_file_path()
+		if os.path.isfile(lock_file):
+			raise IOError("Lock for file %r did already exist, delete %r in case the lock is illegal" % (self._file_path, lock_file))
+
+		try:
+			fd = os.open(lock_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0)
+			os.close(fd)
+		except OSError,e:
+			raise IOError(str(e))
+
+		self._owns_lock = True
+
+	def _obtain_lock(self):
+		"""The default implementation will raise if a lock cannot be obtained.
+		Subclasses may override this method to provide a different implementation"""
+		return self._obtain_lock_or_raise()
+
+	def _release_lock(self):
+		"""Release our lock if we have one"""
+		if not self._has_lock():
+			return
+
+		# if someone removed our file beforhand, lets just flag this issue
+		# instead of failing, to make it more usable.
+		lfp = self._lock_file_path()
+		try:
+			# on bloody windows, the file needs write permissions to be removable.
+			# Why ...
+			if os.name == 'nt':
+				os.chmod(lfp, 0777)
+			# END handle win32
+			os.remove(lfp)
+		except OSError:
+			pass
+		self._owns_lock = False

rhodecode/lib/vcs/utils/ordered_dict.py

@@ -0,0 +1,102 b''
+"""Ordered dict implementation"""
+from UserDict import DictMixin
+
+
+class OrderedDict(dict, DictMixin):
+
+    def __init__(self, *args, **kwds):
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        try:
+            self.__end
+        except AttributeError:
+            self.clear()
+        self.update(*args, **kwds)
+
+    def clear(self):
+        self.__end = end = []
+        end += [None, end, end]         # sentinel node for doubly linked list
+        self.__map = {}                 # key --> [key, prev, next]
+        dict.clear(self)
+
+    def __setitem__(self, key, value):
+        if key not in self:
+            end = self.__end
+            curr = end[1]
+            curr[2] = end[1] = self.__map[key] = [key, curr, end]
+        dict.__setitem__(self, key, value)
+
+    def __delitem__(self, key):
+        dict.__delitem__(self, key)
+        key, prev, next = self.__map.pop(key)
+        prev[2] = next
+        next[1] = prev
+
+    def __iter__(self):
+        end = self.__end
+        curr = end[2]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[2]
+
+    def __reversed__(self):
+        end = self.__end
+        curr = end[1]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[1]
+
+    def popitem(self, last=True):
+        if not self:
+            raise KeyError('dictionary is empty')
+        if last:
+            key = reversed(self).next()
+        else:
+            key = iter(self).next()
+        value = self.pop(key)
+        return key, value
+
+    def __reduce__(self):
+        items = [[k, self[k]] for k in self]
+        tmp = self.__map, self.__end
+        del self.__map, self.__end
+        inst_dict = vars(self).copy()
+        self.__map, self.__end = tmp
+        if inst_dict:
+            return (self.__class__, (items,), inst_dict)
+        return self.__class__, (items,)
+
+    def keys(self):
+        return list(self)
+
+    setdefault = DictMixin.setdefault
+    update = DictMixin.update
+    pop = DictMixin.pop
+    values = DictMixin.values
+    items = DictMixin.items
+    iterkeys = DictMixin.iterkeys
+    itervalues = DictMixin.itervalues
+    iteritems = DictMixin.iteritems
+
+    def __repr__(self):
+        if not self:
+            return '%s()' % (self.__class__.__name__,)
+        return '%s(%r)' % (self.__class__.__name__, self.items())
+
+    def copy(self):
+        return self.__class__(self)
+
+    @classmethod
+    def fromkeys(cls, iterable, value=None):
+        d = cls()
+        for key in iterable:
+            d[key] = value
+        return d
+
+    def __eq__(self, other):
+        if isinstance(other, OrderedDict):
+            return len(self) == len(other) and self.items() == other.items()
+        return dict.__eq__(self, other)
+
+    def __ne__(self, other):
+        return not self == other

rhodecode/lib/vcs/utils/paths.py

@@ -0,0 +1,36 b''
+import os
+
+abspath = lambda * p: os.path.abspath(os.path.join(*p))
+
+
+def get_dirs_for_path(*paths):
+    """
+    Returns list of directories, including intermediate.
+    """
+    for path in paths:
+        head = path
+        while head:
+            head, tail = os.path.split(head)
+            if head:
+                yield head
+            else:
+                # We don't need to yield empty path
+                break
+
+
+def get_dir_size(path):
+    root_path = path
+    size = 0
+    for path, dirs, files in os.walk(root_path):
+        for f in files:
+            try:
+                size += os.path.getsize(os.path.join(path, f))
+            except OSError:
+                pass
+    return size
+
+def get_user_home():
+    """
+    Returns home path of the user.
+    """
+    return os.getenv('HOME', os.getenv('USERPROFILE'))

rhodecode/lib/vcs/utils/progressbar.py

@@ -0,0 +1,419 b''
+# encoding: UTF-8
+import sys
+import datetime
+from string import Template
+from rhodecode.lib.vcs.utils.filesize import filesizeformat
+from rhodecode.lib.vcs.utils.helpers import get_total_seconds
+
+
+class ProgressBarError(Exception):
+    pass
+
+class AlreadyFinishedError(ProgressBarError):
+    pass
+
+
+class ProgressBar(object):
+
+    default_elements = ['percentage', 'bar', 'steps']
+
+    def __init__(self, steps=100, stream=None, elements=None):
+        self.step = 0
+        self.steps = steps
+        self.stream = stream or sys.stderr
+        self.bar_char = '='
+        self.width = 50
+        self.separator = ' | '
+        self.elements = elements or self.default_elements
+        self.started = None
+        self.finished = False
+        self.steps_label = 'Step'
+        self.time_label = 'Time'
+        self.eta_label = 'ETA'
+        self.speed_label = 'Speed'
+        self.transfer_label = 'Transfer'
+
+    def __str__(self):
+        return self.get_line()
+
+    def __iter__(self):
+        start = self.step
+        end = self.steps + 1
+        for x in xrange(start, end):
+            self.render(x)
+            yield x
+
+    def get_separator(self):
+        return self.separator
+
+    def get_bar_char(self):
+        return self.bar_char
+
+    def get_bar(self):
+        char = self.get_bar_char()
+        perc = self.get_percentage()
+        length = int(self.width * perc / 100)
+        bar = char * length
+        bar = bar.ljust(self.width)
+        return bar
+
+    def get_elements(self):
+        return self.elements
+
+    def get_template(self):
+        separator = self.get_separator()
+        elements = self.get_elements()
+        return Template(separator.join((('$%s' % e) for e in elements)))
+
+    def get_total_time(self, current_time=None):
+        if current_time is None:
+            current_time = datetime.datetime.now()
+        if not self.started:
+            return datetime.timedelta()
+        return current_time - self.started
+
+    def get_rendered_total_time(self):
+        delta = self.get_total_time()
+        if not delta:
+            ttime = '-'
+        else:
+            ttime = str(delta)
+        return '%s %s' % (self.time_label, ttime)
+
+    def get_eta(self, current_time=None):
+        if current_time is None:
+            current_time = datetime.datetime.now()
+        if self.step == 0:
+            return datetime.timedelta()
+        total_seconds = get_total_seconds(self.get_total_time())
+        eta_seconds = total_seconds * self.steps / self.step - total_seconds
+        return datetime.timedelta(seconds=int(eta_seconds))
+
+    def get_rendered_eta(self):
+        eta = self.get_eta()
+        if not eta:
+            eta = '--:--:--'
+        else:
+            eta = str(eta).rjust(8)
+        return '%s: %s' % (self.eta_label, eta)
+
+    def get_percentage(self):
+        return float(self.step) / self.steps * 100
+
+    def get_rendered_percentage(self):
+        perc = self.get_percentage()
+        return ('%s%%' % (int(perc))).rjust(5)
+
+    def get_rendered_steps(self):
+        return '%s: %s/%s' % (self.steps_label, self.step, self.steps)
+
+    def get_rendered_speed(self, step=None, total_seconds=None):
+        if step is None:
+            step = self.step
+        if total_seconds is None:
+            total_seconds = get_total_seconds(self.get_total_time())
+        if step <= 0 or total_seconds <= 0:
+            speed = '-'
+        else:
+            speed = filesizeformat(float(step) / total_seconds)
+        return '%s: %s/s' % (self.speed_label, speed)
+
+    def get_rendered_transfer(self, step=None, steps=None):
+        if step is None:
+            step = self.step
+        if steps is None:
+            steps = self.steps
+
+        if steps <= 0:
+            return '%s: -' % self.transfer_label
+        total = filesizeformat(float(steps))
+        if step <= 0:
+            transferred = '-'
+        else:
+            transferred = filesizeformat(float(step))
+        return '%s: %s / %s' % (self.transfer_label, transferred, total)
+
+    def get_context(self):
+        return {
+            'percentage': self.get_rendered_percentage(),
+            'bar': self.get_bar(),
+            'steps': self.get_rendered_steps(),
+            'time': self.get_rendered_total_time(),
+            'eta': self.get_rendered_eta(),
+            'speed': self.get_rendered_speed(),
+            'transfer': self.get_rendered_transfer(),
+        }
+
+    def get_line(self):
+        template = self.get_template()
+        context = self.get_context()
+        return template.safe_substitute(**context)
+
+    def write(self, data):
+        self.stream.write(data)
+
+    def render(self, step):
+        if not self.started:
+            self.started = datetime.datetime.now()
+        if self.finished:
+            raise AlreadyFinishedError
+        self.step = step
+        self.write('\r%s' % self)
+        if step == self.steps:
+            self.finished = True
+        if step == self.steps:
+            self.write('\n')
+
+
+"""
+termcolors.py
+
+Grabbed from Django (http://www.djangoproject.com)
+"""
+
+color_names = ('black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white')
+foreground = dict([(color_names[x], '3%s' % x) for x in range(8)])
+background = dict([(color_names[x], '4%s' % x) for x in range(8)])
+
+RESET = '0'
+opt_dict = {'bold': '1', 'underscore': '4', 'blink': '5', 'reverse': '7', 'conceal': '8'}
+
+def colorize(text='', opts=(), **kwargs):
+    """
+    Returns your text, enclosed in ANSI graphics codes.
+
+    Depends on the keyword arguments 'fg' and 'bg', and the contents of
+    the opts tuple/list.
+
+    Returns the RESET code if no parameters are given.
+
+    Valid colors:
+        'black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white'
+
+    Valid options:
+        'bold'
+        'underscore'
+        'blink'
+        'reverse'
+        'conceal'
+        'noreset' - string will not be auto-terminated with the RESET code
+
+    Examples:
+        colorize('hello', fg='red', bg='blue', opts=('blink',))
+        colorize()
+        colorize('goodbye', opts=('underscore',))
+        print colorize('first line', fg='red', opts=('noreset',))
+        print 'this should be red too'
+        print colorize('and so should this')
+        print 'this should not be red'
+    """
+    code_list = []
+    if text == '' and len(opts) == 1 and opts[0] == 'reset':
+        return '\x1b[%sm' % RESET
+    for k, v in kwargs.iteritems():
+        if k == 'fg':
+            code_list.append(foreground[v])
+        elif k == 'bg':
+            code_list.append(background[v])
+    for o in opts:
+        if o in opt_dict:
+            code_list.append(opt_dict[o])
+    if 'noreset' not in opts:
+        text = text + '\x1b[%sm' % RESET
+    return ('\x1b[%sm' % ';'.join(code_list)) + text
+
+def make_style(opts=(), **kwargs):
+    """
+    Returns a function with default parameters for colorize()
+
+    Example:
+        bold_red = make_style(opts=('bold',), fg='red')
+        print bold_red('hello')
+        KEYWORD = make_style(fg='yellow')
+        COMMENT = make_style(fg='blue', opts=('bold',))
+    """
+    return lambda text: colorize(text, opts, **kwargs)
+
+NOCOLOR_PALETTE = 'nocolor'
+DARK_PALETTE = 'dark'
+LIGHT_PALETTE = 'light'
+
+PALETTES = {
+    NOCOLOR_PALETTE: {
+        'ERROR':        {},
+        'NOTICE':       {},
+        'SQL_FIELD':    {},
+        'SQL_COLTYPE':  {},
+        'SQL_KEYWORD':  {},
+        'SQL_TABLE':    {},
+        'HTTP_INFO':         {},
+        'HTTP_SUCCESS':      {},
+        'HTTP_REDIRECT':     {},
+        'HTTP_NOT_MODIFIED': {},
+        'HTTP_BAD_REQUEST':  {},
+        'HTTP_NOT_FOUND':    {},
+        'HTTP_SERVER_ERROR': {},
+    },
+    DARK_PALETTE: {
+        'ERROR':        { 'fg': 'red', 'opts': ('bold',) },
+        'NOTICE':       { 'fg': 'red' },
+        'SQL_FIELD':    { 'fg': 'green', 'opts': ('bold',) },
+        'SQL_COLTYPE':  { 'fg': 'green' },
+        'SQL_KEYWORD':  { 'fg': 'yellow' },
+        'SQL_TABLE':    { 'opts': ('bold',) },
+        'HTTP_INFO':         { 'opts': ('bold',) },
+        'HTTP_SUCCESS':      { },
+        'HTTP_REDIRECT':     { 'fg': 'green' },
+        'HTTP_NOT_MODIFIED': { 'fg': 'cyan' },
+        'HTTP_BAD_REQUEST':  { 'fg': 'red', 'opts': ('bold',) },
+        'HTTP_NOT_FOUND':    { 'fg': 'yellow' },
+        'HTTP_SERVER_ERROR': { 'fg': 'magenta', 'opts': ('bold',) },
+    },
+    LIGHT_PALETTE: {
+        'ERROR':        { 'fg': 'red', 'opts': ('bold',) },
+        'NOTICE':       { 'fg': 'red' },
+        'SQL_FIELD':    { 'fg': 'green', 'opts': ('bold',) },
+        'SQL_COLTYPE':  { 'fg': 'green' },
+        'SQL_KEYWORD':  { 'fg': 'blue' },
+        'SQL_TABLE':    { 'opts': ('bold',) },
+        'HTTP_INFO':         { 'opts': ('bold',) },
+        'HTTP_SUCCESS':      { },
+        'HTTP_REDIRECT':     { 'fg': 'green', 'opts': ('bold',) },
+        'HTTP_NOT_MODIFIED': { 'fg': 'green' },
+        'HTTP_BAD_REQUEST':  { 'fg': 'red', 'opts': ('bold',) },
+        'HTTP_NOT_FOUND':    { 'fg': 'red' },
+        'HTTP_SERVER_ERROR': { 'fg': 'magenta', 'opts': ('bold',) },
+    }
+}
+DEFAULT_PALETTE = DARK_PALETTE
+
+# ---------------------------- #
+# --- End of termcolors.py --- #
+# ---------------------------- #
+
+
+class ColoredProgressBar(ProgressBar):
+
+    BAR_COLORS = (
+        (10, 'red'),
+        (30, 'magenta'),
+        (50, 'yellow'),
+        (99, 'green'),
+        (100, 'blue'),
+    )
+
+    def get_line(self):
+        line = super(ColoredProgressBar, self).get_line()
+        perc = self.get_percentage()
+        if perc > 100:
+            color = 'blue'
+        for max_perc, color in self.BAR_COLORS:
+            if perc <= max_perc:
+                break
+        return colorize(line, fg=color)
+
+
+class AnimatedProgressBar(ProgressBar):
+
+    def get_bar_char(self):
+        chars = '-/|\\'
+        if self.step >= self.steps:
+            return '='
+        return chars[self.step % len(chars)]
+
+
+class BarOnlyProgressBar(ProgressBar):
+
+    default_elements = ['bar', 'steps']
+
+    def get_bar(self):
+        bar = super(BarOnlyProgressBar, self).get_bar()
+        perc = self.get_percentage()
+        perc_text = '%s%%' % int(perc)
+        text = (' %s%% ' % (perc_text)).center(self.width, '=')
+        L = text.find(' ')
+        R = text.rfind(' ')
+        bar = ' '.join((bar[:L], perc_text, bar[R:]))
+        return bar
+
+
+class AnimatedColoredProgressBar(AnimatedProgressBar,
+                                 ColoredProgressBar):
+    pass
+
+
+class BarOnlyColoredProgressBar(ColoredProgressBar,
+                                BarOnlyProgressBar):
+    pass
+
+
+
+def main():
+    import time
+
+    print "Standard progress bar..."
+    bar = ProgressBar(30)
+    for x in xrange(1, 31):
+            bar.render(x)
+            time.sleep(0.02)
+    bar.stream.write('\n')
+    print
+
+    print "Empty bar..."
+    bar = ProgressBar(50)
+    bar.render(0)
+    print
+    print
+
+    print "Colored bar..."
+    bar = ColoredProgressBar(20)
+    for x in bar:
+        time.sleep(0.01)
+    print
+
+    print "Animated char bar..."
+    bar = AnimatedProgressBar(20)
+    for x in bar:
+        time.sleep(0.01)
+    print
+
+    print "Animated + colored char bar..."
+    bar = AnimatedColoredProgressBar(20)
+    for x in bar:
+        time.sleep(0.01)
+    print
+
+    print "Bar only ..."
+    bar = BarOnlyProgressBar(20)
+    for x in bar:
+        time.sleep(0.01)
+    print
+
+    print "Colored, longer bar-only, eta, total time ..."
+    bar = BarOnlyColoredProgressBar(40)
+    bar.width = 60
+    bar.elements += ['time', 'eta']
+    for x in bar:
+        time.sleep(0.01)
+    print
+    print
+
+    print "File transfer bar, breaks after 2 seconds ..."
+    total_bytes = 1024 * 1024 * 2
+    bar = ProgressBar(total_bytes)
+    bar.width = 50
+    bar.elements.remove('steps')
+    bar.elements += ['transfer', 'time', 'eta', 'speed']
+    for x in xrange(0, bar.steps, 1024):
+        bar.render(x)
+        time.sleep(0.01)
+        now = datetime.datetime.now()
+        if now - bar.started >= datetime.timedelta(seconds=2):
+            break
+    print
+    print
+
+
+
+if __name__ == '__main__':
+    main()

rhodecode/lib/vcs/utils/termcolors.py

@@ -0,0 +1,200 b''
+"""
+termcolors.py
+
+Grabbed from Django (http://www.djangoproject.com)
+"""
+
+color_names = ('black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white')
+foreground = dict([(color_names[x], '3%s' % x) for x in range(8)])
+background = dict([(color_names[x], '4%s' % x) for x in range(8)])
+
+RESET = '0'
+opt_dict = {'bold': '1', 'underscore': '4', 'blink': '5', 'reverse': '7', 'conceal': '8'}
+
+def colorize(text='', opts=(), **kwargs):
+    """
+    Returns your text, enclosed in ANSI graphics codes.
+
+    Depends on the keyword arguments 'fg' and 'bg', and the contents of
+    the opts tuple/list.
+
+    Returns the RESET code if no parameters are given.
+
+    Valid colors:
+        'black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white'
+
+    Valid options:
+        'bold'
+        'underscore'
+        'blink'
+        'reverse'
+        'conceal'
+        'noreset' - string will not be auto-terminated with the RESET code
+
+    Examples:
+        colorize('hello', fg='red', bg='blue', opts=('blink',))
+        colorize()
+        colorize('goodbye', opts=('underscore',))
+        print colorize('first line', fg='red', opts=('noreset',))
+        print 'this should be red too'
+        print colorize('and so should this')
+        print 'this should not be red'
+    """
+    code_list = []
+    if text == '' and len(opts) == 1 and opts[0] == 'reset':
+        return '\x1b[%sm' % RESET
+    for k, v in kwargs.iteritems():
+        if k == 'fg':
+            code_list.append(foreground[v])
+        elif k == 'bg':
+            code_list.append(background[v])
+    for o in opts:
+        if o in opt_dict:
+            code_list.append(opt_dict[o])
+    if 'noreset' not in opts:
+        text = text + '\x1b[%sm' % RESET
+    return ('\x1b[%sm' % ';'.join(code_list)) + text
+
+def make_style(opts=(), **kwargs):
+    """
+    Returns a function with default parameters for colorize()
+
+    Example:
+        bold_red = make_style(opts=('bold',), fg='red')
+        print bold_red('hello')
+        KEYWORD = make_style(fg='yellow')
+        COMMENT = make_style(fg='blue', opts=('bold',))
+    """
+    return lambda text: colorize(text, opts, **kwargs)
+
+NOCOLOR_PALETTE = 'nocolor'
+DARK_PALETTE = 'dark'
+LIGHT_PALETTE = 'light'
+
+PALETTES = {
+    NOCOLOR_PALETTE: {
+        'ERROR':        {},
+        'NOTICE':       {},
+        'SQL_FIELD':    {},
+        'SQL_COLTYPE':  {},
+        'SQL_KEYWORD':  {},
+        'SQL_TABLE':    {},
+        'HTTP_INFO':         {},
+        'HTTP_SUCCESS':      {},
+        'HTTP_REDIRECT':     {},
+        'HTTP_NOT_MODIFIED': {},
+        'HTTP_BAD_REQUEST':  {},
+        'HTTP_NOT_FOUND':    {},
+        'HTTP_SERVER_ERROR': {},
+    },
+    DARK_PALETTE: {
+        'ERROR':        { 'fg': 'red', 'opts': ('bold',) },
+        'NOTICE':       { 'fg': 'red' },
+        'SQL_FIELD':    { 'fg': 'green', 'opts': ('bold',) },
+        'SQL_COLTYPE':  { 'fg': 'green' },
+        'SQL_KEYWORD':  { 'fg': 'yellow' },
+        'SQL_TABLE':    { 'opts': ('bold',) },
+        'HTTP_INFO':         { 'opts': ('bold',) },
+        'HTTP_SUCCESS':      { },
+        'HTTP_REDIRECT':     { 'fg': 'green' },
+        'HTTP_NOT_MODIFIED': { 'fg': 'cyan' },
+        'HTTP_BAD_REQUEST':  { 'fg': 'red', 'opts': ('bold',) },
+        'HTTP_NOT_FOUND':    { 'fg': 'yellow' },
+        'HTTP_SERVER_ERROR': { 'fg': 'magenta', 'opts': ('bold',) },
+    },
+    LIGHT_PALETTE: {
+        'ERROR':        { 'fg': 'red', 'opts': ('bold',) },
+        'NOTICE':       { 'fg': 'red' },
+        'SQL_FIELD':    { 'fg': 'green', 'opts': ('bold',) },
+        'SQL_COLTYPE':  { 'fg': 'green' },
+        'SQL_KEYWORD':  { 'fg': 'blue' },
+        'SQL_TABLE':    { 'opts': ('bold',) },
+        'HTTP_INFO':         { 'opts': ('bold',) },
+        'HTTP_SUCCESS':      { },
+        'HTTP_REDIRECT':     { 'fg': 'green', 'opts': ('bold',) },
+        'HTTP_NOT_MODIFIED': { 'fg': 'green' },
+        'HTTP_BAD_REQUEST':  { 'fg': 'red', 'opts': ('bold',) },
+        'HTTP_NOT_FOUND':    { 'fg': 'red' },
+        'HTTP_SERVER_ERROR': { 'fg': 'magenta', 'opts': ('bold',) },
+    }
+}
+DEFAULT_PALETTE = DARK_PALETTE
+
+def parse_color_setting(config_string):
+    """Parse a DJANGO_COLORS environment variable to produce the system palette
+
+    The general form of a pallete definition is:
+
+        "palette;role=fg;role=fg/bg;role=fg,option,option;role=fg/bg,option,option"
+
+    where:
+        palette is a named palette; one of 'light', 'dark', or 'nocolor'.
+        role is a named style used by Django
+        fg is a background color.
+        bg is a background color.
+        option is a display options.
+
+    Specifying a named palette is the same as manually specifying the individual
+    definitions for each role. Any individual definitions following the pallete
+    definition will augment the base palette definition.
+
+    Valid roles:
+        'error', 'notice', 'sql_field', 'sql_coltype', 'sql_keyword', 'sql_table',
+        'http_info', 'http_success', 'http_redirect', 'http_bad_request',
+        'http_not_found', 'http_server_error'
+
+    Valid colors:
+        'black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white'
+
+    Valid options:
+        'bold', 'underscore', 'blink', 'reverse', 'conceal'
+
+    """
+    if not config_string:
+        return PALETTES[DEFAULT_PALETTE]
+
+    # Split the color configuration into parts
+    parts = config_string.lower().split(';')
+    palette = PALETTES[NOCOLOR_PALETTE].copy()
+    for part in parts:
+        if part in PALETTES:
+            # A default palette has been specified
+            palette.update(PALETTES[part])
+        elif '=' in part:
+            # Process a palette defining string
+            definition = {}
+
+            # Break the definition into the role,
+            # plus the list of specific instructions.
+            # The role must be in upper case
+            role, instructions = part.split('=')
+            role = role.upper()
+
+            styles = instructions.split(',')
+            styles.reverse()
+
+            # The first instruction can contain a slash
+            # to break apart fg/bg.
+            colors = styles.pop().split('/')
+            colors.reverse()
+            fg = colors.pop()
+            if fg in color_names:
+                definition['fg'] = fg
+            if colors and colors[-1] in color_names:
+                definition['bg'] = colors[-1]
+
+            # All remaining instructions are options
+            opts = tuple(s for s in styles if s in opt_dict.keys())
+            if opts:
+                definition['opts'] = opts
+
+            # The nocolor palette has all available roles.
+            # Use that palette as the basis for determining
+            # if the role is valid.
+            if role in PALETTES[NOCOLOR_PALETTE] and definition:
+                palette[role] = definition
+
+    # If there are no colors specified, return the empty palette.
+    if palette == PALETTES[NOCOLOR_PALETTE]:
+        return None
+    return palette

.hgignore

@@ -1,19 +1,20 b''
 syntax: glob
 *.pyc
 *.swp
 *.sqlite
 Paste*.egg
 
 syntax: regexp
 ^build
 ^docs/build/
 ^docs/_build/
 ^data$
 ^\.settings$
 ^\.project$
 ^\.pydevproject$
 ^\.coverage$
 ^rhodecode\.db$
 ^test\.db$
 ^RhodeCode\.egg-info$
 ^rc\.ini$
+^fabfile.py

requires.txt

@@ -1,18 +1,17 b''
 Pylons==1.0.0
 Beaker==1.6.2
 WebHelpers>=1.2
 formencode==1.2.4
 SQLAlchemy==0.7.4
 Mako==0.5.0
 pygments>=1.4
 whoosh>=2.3.0,<2.4
 celery>=2.2.5,<2.3
 babel
 python-dateutil>=1.5.0,<2.0.0
 dulwich>=0.8.0,<0.9.0
-vcs>=0.2.3.dev
 webob==1.0.8
-markdown==2.0.3
+markdown==2.1.1
 docutils==0.8.1
 py-bcrypt
 mercurial>=2.1,<2.2
  No newline at end of file

rhodecode/__init__.py

@@ -1,93 +1,92 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.__init__
     ~~~~~~~~~~~~~~~~~~
 
     RhodeCode, a web based repository management based on pylons
     versioning implementation: http://semver.org/
 
     :created_on: Apr 9, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import sys
 import platform
 
 VERSION = (1, 3, 0, 'beta')
 __version__ = '.'.join((str(each) for each in VERSION[:4]))
 __dbversion__ = 5  # defines current db version for migrations
 __platform__ = platform.system()
 __license__ = 'GPLv3'
 __py_version__ = sys.version_info
 
 PLATFORM_WIN = ('Windows')
 PLATFORM_OTHERS = ('Linux', 'Darwin', 'FreeBSD', 'OpenBSD', 'SunOS')
 
 requirements = [
     "Pylons==1.0.0",
     "Beaker==1.6.2",
     "WebHelpers>=1.2",
     "formencode==1.2.4",
     "SQLAlchemy==0.7.4",
     "Mako==0.5.0",
     "pygments>=1.4",
     "whoosh>=2.3.0,<2.4",
     "celery>=2.2.5,<2.3",
     "babel",
     "python-dateutil>=1.5.0,<2.0.0",
     "dulwich>=0.8.0,<0.9.0",
-    "vcs>=0.2.3.dev",
     "webob==1.0.8",
-    "markdown==2.0.3",
+    "markdown==2.1.1",
     "docutils==0.8.1",
 ]
 
 if __py_version__ < (2, 6):
     requirements.append("simplejson")
     requirements.append("pysqlite")
 
 if __platform__ in PLATFORM_WIN:
     requirements.append("mercurial>=2.1,<2.2")
 else:
     requirements.append("py-bcrypt")
     requirements.append("mercurial>=2.1,<2.2")
 
 
 try:
     from rhodecode.lib import get_current_revision
     _rev = get_current_revision()
 except ImportError:
     # this is needed when doing some setup.py operations
     _rev = False
 
 if len(VERSION) > 3 and _rev:
     __version__ += ' [rev:%s]' % _rev[0]
 
 
 def get_version():
     """Returns shorter version (digit parts only) as string."""
 
     return '.'.join((str(each) for each in VERSION[:3]))
 
 BACKENDS = {
     'hg': 'Mercurial repository',
     'git': 'Git repository',
 }
 
 CELERY_ON = False
 
 # link to config for pylons
 CONFIG = None

rhodecode/controllers/changelog.py

@@ -1,134 +1,134 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.changelog
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     changelog controller for rhodecode
 
     :created_on: Apr 21, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import logging
 import traceback
 
 from mercurial import graphmod
 from pylons import request, url, session, tmpl_context as c
 from pylons.controllers.util import redirect
 from pylons.i18n.translation import _
 
 import rhodecode.lib.helpers as h
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.lib.helpers import RepoPage
 from rhodecode.lib.compat import json
 
-from vcs.exceptions import RepositoryError, ChangesetDoesNotExistError
+from rhodecode.lib.vcs.exceptions import RepositoryError, ChangesetDoesNotExistError
 from rhodecode.model.db import Repository
 
 log = logging.getLogger(__name__)
 
 
 class ChangelogController(BaseRepoController):
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def __before__(self):
         super(ChangelogController, self).__before__()
         c.affected_files_cut_off = 60
 
     def index(self):
         limit = 100
         default = 20
         if request.params.get('size'):
             try:
                 int_size = int(request.params.get('size'))
             except ValueError:
                 int_size = default
             int_size = int_size if int_size <= limit else limit
             c.size = int_size
             session['changelog_size'] = c.size
             session.save()
         else:
             c.size = int(session.get('changelog_size', default))
 
         p = int(request.params.get('page', 1))
         branch_name = request.params.get('branch', None)
         try:
             if branch_name:
                 collection = [z for z in
                               c.rhodecode_repo.get_changesets(start=0,
                                                     branch_name=branch_name)]
                 c.total_cs = len(collection)
             else:
                 collection = c.rhodecode_repo
                 c.total_cs = len(c.rhodecode_repo)
 
             c.pagination = RepoPage(collection, page=p, item_count=c.total_cs,
                                     items_per_page=c.size, branch=branch_name)
             collection = list(c.pagination)
             page_revisions = [x.raw_id for x in collection]
             c.comments = c.rhodecode_db_repo.comments(page_revisions)
 
         except (RepositoryError, ChangesetDoesNotExistError, Exception), e:
             log.error(traceback.format_exc())
             h.flash(str(e), category='warning')
             return redirect(url('home'))
 
         self._graph(c.rhodecode_repo, collection, c.total_cs, c.size, p)
 
         c.branch_name = branch_name
         c.branch_filters = [('', _('All Branches'))] + \
             [(k, k) for k in c.rhodecode_repo.branches.keys()]
 
         return render('changelog/changelog.html')
 
     def changelog_details(self, cs):
         if request.environ.get('HTTP_X_PARTIAL_XHR'):
             c.cs = c.rhodecode_repo.get_changeset(cs)
             return render('changelog/changelog_details.html')
 
     def _graph(self, repo, collection, repo_size, size, p):
         """
         Generates a DAG graph for mercurial
 
         :param repo: repo instance
         :param size: number of commits to show
         :param p: page number
         """
         if not collection:
             c.jsdata = json.dumps([])
             return
 
         data = []
         revs = [x.revision for x in collection]
 
         if repo.alias == 'git':
             for _ in revs:
                 vtx = [0, 1]
                 edges = [[0, 0, 1]]
                 data.append(['', vtx, edges])
 
         elif repo.alias == 'hg':
             c.dag = graphmod.colored(graphmod.dagwalker(repo._repo, revs))
             for (id, type, ctx, vtx, edges) in c.dag:
                 if type != graphmod.CHANGESET:
                     continue
                 data.append(['', vtx, edges])
 
         c.jsdata = json.dumps(data)

rhodecode/controllers/changeset.py

@@ -1,367 +1,367 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.changeset
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     changeset controller for pylons showoing changes beetween
     revisions
 
     :created_on: Apr 25, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import logging
 import traceback
 from collections import defaultdict
 from webob.exc import HTTPForbidden
 
 from pylons import tmpl_context as c, url, request, response
 from pylons.i18n.translation import _
 from pylons.controllers.util import redirect
 from pylons.decorators import jsonify
 
-from vcs.exceptions import RepositoryError, ChangesetError, \
+from rhodecode.lib.vcs.exceptions import RepositoryError, ChangesetError, \
     ChangesetDoesNotExistError
-from vcs.nodes import FileNode
+from rhodecode.lib.vcs.nodes import FileNode
 
 import rhodecode.lib.helpers as h
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.lib.utils import EmptyChangeset
 from rhodecode.lib.compat import OrderedDict
 from rhodecode.lib import diffs
 from rhodecode.model.db import ChangesetComment
 from rhodecode.model.comment import ChangesetCommentsModel
 from rhodecode.model.meta import Session
 from rhodecode.lib.diffs import wrapped_diff
 
 log = logging.getLogger(__name__)
 
 
 def anchor_url(revision, path):
     fid = h.FID(revision, path)
     return h.url.current(anchor=fid, **request.GET)
 
 
 def get_ignore_ws(fid, GET):
     ig_ws_global = request.GET.get('ignorews')
     ig_ws = filter(lambda k: k.startswith('WS'), GET.getall(fid))
     if ig_ws:
         try:
             return int(ig_ws[0].split(':')[-1])
         except:
             pass
     return ig_ws_global
 
 
 def _ignorews_url(fileid=None):
 
     params = defaultdict(list)
     lbl = _('show white space')
     ig_ws = get_ignore_ws(fileid, request.GET)
     ln_ctx = get_line_ctx(fileid, request.GET)
     # global option
     if fileid is None:
         if ig_ws is None:
             params['ignorews'] += [1]
             lbl = _('ignore white space')
         ctx_key = 'context'
         ctx_val = ln_ctx
     # per file options
     else:
         if ig_ws is None:
             params[fileid] += ['WS:1']
             lbl = _('ignore white space')
 
         ctx_key = fileid
         ctx_val = 'C:%s' % ln_ctx
     # if we have passed in ln_ctx pass it along to our params
     if ln_ctx:
         params[ctx_key] += [ctx_val]
 
     params['anchor'] = fileid
     img = h.image('/images/icons/text_strikethrough.png', lbl, class_='icon')
     return h.link_to(img, h.url.current(**params), title=lbl, class_='tooltip')
 
 
 def get_line_ctx(fid, GET):
     ln_ctx_global = request.GET.get('context')
     ln_ctx = filter(lambda k: k.startswith('C'), GET.getall(fid))
 
     if ln_ctx:
         retval = ln_ctx[0].split(':')[-1]
     else:
         retval = ln_ctx_global
 
     try:
         return int(retval)
     except:
         return
 
 
 def _context_url(fileid=None):
     """
     Generates url for context lines
 
     :param fileid:
     """
     ig_ws = get_ignore_ws(fileid, request.GET)
     ln_ctx = (get_line_ctx(fileid, request.GET) or 3) * 2
 
     params = defaultdict(list)
 
     # global option
     if fileid is None:
         if ln_ctx > 0:
             params['context'] += [ln_ctx]
 
         if ig_ws:
             ig_ws_key = 'ignorews'
             ig_ws_val = 1
 
     # per file option
     else:
         params[fileid] += ['C:%s' % ln_ctx]
         ig_ws_key = fileid
         ig_ws_val = 'WS:%s' % 1
 
     if ig_ws:
         params[ig_ws_key] += [ig_ws_val]
 
     lbl = _('%s line context') % ln_ctx
 
     params['anchor'] = fileid
     img = h.image('/images/icons/table_add.png', lbl, class_='icon')
     return h.link_to(img, h.url.current(**params), title=lbl, class_='tooltip')
 
 
 class ChangesetController(BaseRepoController):
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def __before__(self):
         super(ChangesetController, self).__before__()
         c.affected_files_cut_off = 60
 
     def index(self, revision):
 
         c.anchor_url = anchor_url
         c.ignorews_url = _ignorews_url
         c.context_url = _context_url
 
         #get ranges of revisions if preset
         rev_range = revision.split('...')[:2]
         enable_comments = True
         try:
             if len(rev_range) == 2:
                 enable_comments = False
                 rev_start = rev_range[0]
                 rev_end = rev_range[1]
                 rev_ranges = c.rhodecode_repo.get_changesets(start=rev_start,
                                                             end=rev_end)
             else:
                 rev_ranges = [c.rhodecode_repo.get_changeset(revision)]
 
             c.cs_ranges = list(rev_ranges)
             if not c.cs_ranges:
                 raise RepositoryError('Changeset range returned empty result')
 
         except (RepositoryError, ChangesetDoesNotExistError, Exception), e:
             log.error(traceback.format_exc())
             h.flash(str(e), category='warning')
             return redirect(url('home'))
 
         c.changes = OrderedDict()
 
         c.lines_added = 0  # count of lines added
         c.lines_deleted = 0  # count of lines removes
 
         cumulative_diff = 0
         c.cut_off = False  # defines if cut off limit is reached
 
         c.comments = []
         c.inline_comments = []
         c.inline_cnt = 0
         # Iterate over ranges (default changeset view is always one changeset)
         for changeset in c.cs_ranges:
             c.comments.extend(ChangesetCommentsModel()\
                               .get_comments(c.rhodecode_db_repo.repo_id,
                                             changeset.raw_id))
             inlines = ChangesetCommentsModel()\
                         .get_inline_comments(c.rhodecode_db_repo.repo_id,
                                              changeset.raw_id)
             c.inline_comments.extend(inlines)
             c.changes[changeset.raw_id] = []
             try:
                 changeset_parent = changeset.parents[0]
             except IndexError:
                 changeset_parent = None
 
             #==================================================================
             # ADDED FILES
             #==================================================================
             for node in changeset.added:
                 fid = h.FID(revision, node.path)
                 line_context_lcl = get_line_ctx(fid, request.GET)
                 ign_whitespace_lcl = get_ignore_ws(fid, request.GET)
                 lim = self.cut_off_limit
                 if cumulative_diff > self.cut_off_limit:
                     lim = -1
                 size, cs1, cs2, diff, st = wrapped_diff(filenode_old=None,
                                          filenode_new=node,
                                          cut_off_limit=lim,
                                          ignore_whitespace=ign_whitespace_lcl,
                                          line_context=line_context_lcl,
                                          enable_comments=enable_comments)
                 cumulative_diff += size
                 c.lines_added += st[0]
                 c.lines_deleted += st[1]
                 c.changes[changeset.raw_id].append(('added', node, diff,
                                                     cs1, cs2, st))
 
             #==================================================================
             # CHANGED FILES
             #==================================================================
             for node in changeset.changed:
                 try:
                     filenode_old = changeset_parent.get_node(node.path)
                 except ChangesetError:
                     log.warning('Unable to fetch parent node for diff')
                     filenode_old = FileNode(node.path, '', EmptyChangeset())
 
                 fid = h.FID(revision, node.path)
                 line_context_lcl = get_line_ctx(fid, request.GET)
                 ign_whitespace_lcl = get_ignore_ws(fid, request.GET)
                 lim = self.cut_off_limit
                 if cumulative_diff > self.cut_off_limit:
                     lim = -1
                 size, cs1, cs2, diff, st = wrapped_diff(filenode_old=filenode_old,
                                          filenode_new=node,
                                          cut_off_limit=lim,
                                          ignore_whitespace=ign_whitespace_lcl,
                                          line_context=line_context_lcl,
                                          enable_comments=enable_comments)
                 cumulative_diff += size
                 c.lines_added += st[0]
                 c.lines_deleted += st[1]
                 c.changes[changeset.raw_id].append(('changed', node, diff,
                                                     cs1, cs2, st))
 
             #==================================================================
             # REMOVED FILES
             #==================================================================
             for node in changeset.removed:
                 c.changes[changeset.raw_id].append(('removed', node, None,
                                                     None, None, (0, 0)))
 
         # count inline comments
         for path, lines in c.inline_comments:
             for comments in lines.values():
                 c.inline_cnt += len(comments)
 
         if len(c.cs_ranges) == 1:
             c.changeset = c.cs_ranges[0]
             c.changes = c.changes[c.changeset.raw_id]
 
             return render('changeset/changeset.html')
         else:
             return render('changeset/changeset_range.html')
 
     def raw_changeset(self, revision):
 
         method = request.GET.get('diff', 'show')
         ignore_whitespace = request.GET.get('ignorews') == '1'
         line_context = request.GET.get('context', 3)
         try:
             c.scm_type = c.rhodecode_repo.alias
             c.changeset = c.rhodecode_repo.get_changeset(revision)
         except RepositoryError:
             log.error(traceback.format_exc())
             return redirect(url('home'))
         else:
             try:
                 c.changeset_parent = c.changeset.parents[0]
             except IndexError:
                 c.changeset_parent = None
             c.changes = []
 
             for node in c.changeset.added:
                 filenode_old = FileNode(node.path, '')
                 if filenode_old.is_binary or node.is_binary:
                     diff = _('binary file') + '\n'
                 else:
                     f_gitdiff = diffs.get_gitdiff(filenode_old, node,
                                            ignore_whitespace=ignore_whitespace,
                                            context=line_context)
                     diff = diffs.DiffProcessor(f_gitdiff,
                                                 format='gitdiff').raw_diff()
 
                 cs1 = None
                 cs2 = node.last_changeset.raw_id
                 c.changes.append(('added', node, diff, cs1, cs2))
 
             for node in c.changeset.changed:
                 filenode_old = c.changeset_parent.get_node(node.path)
                 if filenode_old.is_binary or node.is_binary:
                     diff = _('binary file')
                 else:
                     f_gitdiff = diffs.get_gitdiff(filenode_old, node,
                                            ignore_whitespace=ignore_whitespace,
                                            context=line_context)
                     diff = diffs.DiffProcessor(f_gitdiff,
                                                 format='gitdiff').raw_diff()
 
                 cs1 = filenode_old.last_changeset.raw_id
                 cs2 = node.last_changeset.raw_id
                 c.changes.append(('changed', node, diff, cs1, cs2))
 
         response.content_type = 'text/plain'
 
         if method == 'download':
             response.content_disposition = 'attachment; filename=%s.patch' \
                                             % revision
 
         c.parent_tmpl = ''.join(['# Parent  %s\n' % x.raw_id for x in
                                                  c.changeset.parents])
 
         c.diffs = ''
         for x in c.changes:
             c.diffs += x[2]
 
         return render('changeset/raw_changeset.html')
 
     def comment(self, repo_name, revision):
         ChangesetCommentsModel().create(text=request.POST.get('text'),
                                         repo_id=c.rhodecode_db_repo.repo_id,
                                         user_id=c.rhodecode_user.user_id,
                                         revision=revision,
                                         f_path=request.POST.get('f_path'),
                                         line_no=request.POST.get('line'))
         Session.commit()
         return redirect(h.url('changeset_home', repo_name=repo_name,
                               revision=revision))
 
     @jsonify
     def delete_comment(self, repo_name, comment_id):
         co = ChangesetComment.get(comment_id)
         owner = lambda: co.author.user_id == c.rhodecode_user.user_id
         if h.HasPermissionAny('hg.admin', 'repository.admin')() or owner:
             ChangesetCommentsModel().delete(comment=co)
             Session.commit()
             return True
         else:
             raise HTTPForbidden()

rhodecode/controllers/files.py

@@ -1,492 +1,492 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.files
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Files controller for RhodeCode
 
     :created_on: Apr 21, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import logging
 import traceback
 
 from pylons import request, response, tmpl_context as c, url
 from pylons.i18n.translation import _
 from pylons.controllers.util import redirect
 from pylons.decorators import jsonify
 
-from vcs.conf import settings
-from vcs.exceptions import RepositoryError, ChangesetDoesNotExistError, \
+from rhodecode.lib.vcs.conf import settings
+from rhodecode.lib.vcs.exceptions import RepositoryError, ChangesetDoesNotExistError, \
     EmptyRepositoryError, ImproperArchiveTypeError, VCSError, \
     NodeAlreadyExistsError
-from vcs.nodes import FileNode
+from rhodecode.lib.vcs.nodes import FileNode
 
 from rhodecode.lib.compat import OrderedDict
 from rhodecode.lib import convert_line_endings, detect_mode, safe_str
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.lib.utils import EmptyChangeset
 from rhodecode.lib import diffs
 import rhodecode.lib.helpers as h
 from rhodecode.model.repo import RepoModel
 from rhodecode.controllers.changeset import anchor_url, _ignorews_url,\
     _context_url, get_line_ctx, get_ignore_ws
 from rhodecode.lib.diffs import wrapped_diff
 from rhodecode.model.scm import ScmModel
 
 log = logging.getLogger(__name__)
 
 
 class FilesController(BaseRepoController):
 
     @LoginRequired()
     def __before__(self):
         super(FilesController, self).__before__()
         c.cut_off_limit = self.cut_off_limit
 
     def __get_cs_or_redirect(self, rev, repo_name, redirect_after=True):
         """
         Safe way to get changeset if error occur it redirects to tip with
         proper message
 
         :param rev: revision to fetch
         :param repo_name: repo name to redirect after
         """
 
         try:
             return c.rhodecode_repo.get_changeset(rev)
         except EmptyRepositoryError, e:
             if not redirect_after:
                 return None
             url_ = url('files_add_home',
                        repo_name=c.repo_name,
                        revision=0, f_path='')
             add_new = '<a href="%s">[%s]</a>' % (url_, _('add new'))
             h.flash(h.literal(_('There are no files yet %s' % add_new)),
                     category='warning')
             redirect(h.url('summary_home', repo_name=repo_name))
 
         except RepositoryError, e:
             h.flash(str(e), category='warning')
             redirect(h.url('files_home', repo_name=repo_name, revision='tip'))
 
     def __get_filenode_or_redirect(self, repo_name, cs, path):
         """
         Returns file_node, if error occurs or given path is directory,
         it'll redirect to top level path
 
         :param repo_name: repo_name
         :param cs: given changeset
         :param path: path to lookup
         """
 
         try:
             file_node = cs.get_node(path)
             if file_node.is_dir():
                 raise RepositoryError('given path is a directory')
         except RepositoryError, e:
             h.flash(str(e), category='warning')
             redirect(h.url('files_home', repo_name=repo_name,
                            revision=cs.raw_id))
 
         return file_node
 
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def index(self, repo_name, revision, f_path):
         # redirect to given revision from form if given
         post_revision = request.POST.get('at_rev', None)
         if post_revision:
             cs = self.__get_cs_or_redirect(post_revision, repo_name)
             redirect(url('files_home', repo_name=c.repo_name,
                          revision=cs.raw_id, f_path=f_path))
 
         c.changeset = self.__get_cs_or_redirect(revision, repo_name)
         c.branch = request.GET.get('branch', None)
         c.f_path = f_path
 
         cur_rev = c.changeset.revision
 
         # prev link
         try:
             prev_rev = c.rhodecode_repo.get_changeset(cur_rev).prev(c.branch)
             c.url_prev = url('files_home', repo_name=c.repo_name,
                          revision=prev_rev.raw_id, f_path=f_path)
             if c.branch:
                 c.url_prev += '?branch=%s' % c.branch
         except (ChangesetDoesNotExistError, VCSError):
             c.url_prev = '#'
 
         # next link
         try:
             next_rev = c.rhodecode_repo.get_changeset(cur_rev).next(c.branch)
             c.url_next = url('files_home', repo_name=c.repo_name,
                      revision=next_rev.raw_id, f_path=f_path)
             if c.branch:
                 c.url_next += '?branch=%s' % c.branch
         except (ChangesetDoesNotExistError, VCSError):
             c.url_next = '#'
 
         # files or dirs
         try:
             c.file = c.changeset.get_node(f_path)
 
             if c.file.is_file():
                 c.file_history = self._get_node_history(c.changeset, f_path)
             else:
                 c.file_history = []
         except RepositoryError, e:
             h.flash(str(e), category='warning')
             redirect(h.url('files_home', repo_name=repo_name,
                            revision=revision))
 
         return render('files/files.html')
 
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def rawfile(self, repo_name, revision, f_path):
         cs = self.__get_cs_or_redirect(revision, repo_name)
         file_node = self.__get_filenode_or_redirect(repo_name, cs, f_path)
 
         response.content_disposition = 'attachment; filename=%s' % \
             safe_str(f_path.split(os.sep)[-1])
 
         response.content_type = file_node.mimetype
         return file_node.content
 
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def raw(self, repo_name, revision, f_path):
         cs = self.__get_cs_or_redirect(revision, repo_name)
         file_node = self.__get_filenode_or_redirect(repo_name, cs, f_path)
 
         raw_mimetype_mapping = {
             # map original mimetype to a mimetype used for "show as raw"
             # you can also provide a content-disposition to override the
             # default "attachment" disposition.
             # orig_type: (new_type, new_dispo)
 
             # show images inline:
             'image/x-icon': ('image/x-icon', 'inline'),
             'image/png': ('image/png', 'inline'),
             'image/gif': ('image/gif', 'inline'),
             'image/jpeg': ('image/jpeg', 'inline'),
             'image/svg+xml': ('image/svg+xml', 'inline'),
         }
 
         mimetype = file_node.mimetype
         try:
             mimetype, dispo = raw_mimetype_mapping[mimetype]
         except KeyError:
             # we don't know anything special about this, handle it safely
             if file_node.is_binary:
                 # do same as download raw for binary files
                 mimetype, dispo = 'application/octet-stream', 'attachment'
             else:
                 # do not just use the original mimetype, but force text/plain,
                 # otherwise it would serve text/html and that might be unsafe.
                 # Note: underlying vcs library fakes text/plain mimetype if the
                 # mimetype can not be determined and it thinks it is not
                 # binary.This might lead to erroneous text display in some
                 # cases, but helps in other cases, like with text files
                 # without extension.
                 mimetype, dispo = 'text/plain', 'inline'
 
         if dispo == 'attachment':
             dispo = 'attachment; filename=%s' % \
                         safe_str(f_path.split(os.sep)[-1])
 
         response.content_disposition = dispo
         response.content_type = mimetype
         return file_node.content
 
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def annotate(self, repo_name, revision, f_path):
         c.cs = self.__get_cs_or_redirect(revision, repo_name)
         c.file = self.__get_filenode_or_redirect(repo_name, c.cs, f_path)
 
         c.file_history = self._get_node_history(c.cs, f_path)
         c.f_path = f_path
         return render('files/files_annotate.html')
 
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     def edit(self, repo_name, revision, f_path):
         r_post = request.POST
 
         c.cs = self.__get_cs_or_redirect(revision, repo_name)
         c.file = self.__get_filenode_or_redirect(repo_name, c.cs, f_path)
 
         if c.file.is_binary:
             return redirect(url('files_home', repo_name=c.repo_name,
                          revision=c.cs.raw_id, f_path=f_path))
 
         c.f_path = f_path
 
         if r_post:
 
             old_content = c.file.content
             sl = old_content.splitlines(1)
             first_line = sl[0] if sl else ''
             # modes:  0 - Unix, 1 - Mac, 2 - DOS
             mode = detect_mode(first_line, 0)
             content = convert_line_endings(r_post.get('content'), mode)
 
             message = r_post.get('message') or (_('Edited %s via RhodeCode')
                                                 % (f_path))
             author = self.rhodecode_user.full_contact
 
             if content == old_content:
                 h.flash(_('No changes'),
                     category='warning')
                 return redirect(url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
 
             try:
                 self.scm_model.commit_change(repo=c.rhodecode_repo,
                                              repo_name=repo_name, cs=c.cs,
                                              user=self.rhodecode_user,
                                              author=author, message=message,
                                              content=content, f_path=f_path)
                 h.flash(_('Successfully committed to %s' % f_path),
                         category='success')
 
             except Exception:
                 log.error(traceback.format_exc())
                 h.flash(_('Error occurred during commit'), category='error')
             return redirect(url('changeset_home',
                                 repo_name=c.repo_name, revision='tip'))
 
         return render('files/files_edit.html')
 
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     def add(self, repo_name, revision, f_path):
         r_post = request.POST
         c.cs = self.__get_cs_or_redirect(revision, repo_name,
                                          redirect_after=False)
         if c.cs is None:
             c.cs = EmptyChangeset(alias=c.rhodecode_repo.alias)
 
         c.f_path = f_path
 
         if r_post:
             unix_mode = 0
             content = convert_line_endings(r_post.get('content'), unix_mode)
 
             message = r_post.get('message') or (_('Added %s via RhodeCode')
                                                 % (f_path))
             location = r_post.get('location')
             filename = r_post.get('filename')
             file_obj = r_post.get('upload_file', None)
 
             if file_obj is not None and hasattr(file_obj, 'filename'):
                 filename = file_obj.filename
                 content = file_obj.file
 
             node_path = os.path.join(location, filename)
             author = self.rhodecode_user.full_contact
 
             if not content:
                 h.flash(_('No content'), category='warning')
                 return redirect(url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
             if not filename:
                 h.flash(_('No filename'), category='warning')
                 return redirect(url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
 
             try:
                 self.scm_model.create_node(repo=c.rhodecode_repo,
                                              repo_name=repo_name, cs=c.cs,
                                              user=self.rhodecode_user,
                                              author=author, message=message,
                                              content=content, f_path=node_path)
                 h.flash(_('Successfully committed to %s' % node_path),
                         category='success')
             except NodeAlreadyExistsError, e:
                 h.flash(_(e), category='error')
             except Exception:
                 log.error(traceback.format_exc())
                 h.flash(_('Error occurred during commit'), category='error')
             return redirect(url('changeset_home',
                                 repo_name=c.repo_name, revision='tip'))
 
         return render('files/files_add.html')
 
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def archivefile(self, repo_name, fname):
 
         fileformat = None
         revision = None
         ext = None
         subrepos = request.GET.get('subrepos') == 'true'
 
         for a_type, ext_data in settings.ARCHIVE_SPECS.items():
             archive_spec = fname.split(ext_data[1])
             if len(archive_spec) == 2 and archive_spec[1] == '':
                 fileformat = a_type or ext_data[1]
                 revision = archive_spec[0]
                 ext = ext_data[1]
 
         try:
             dbrepo = RepoModel().get_by_repo_name(repo_name)
             if dbrepo.enable_downloads is False:
                 return _('downloads disabled')
 
             if c.rhodecode_repo.alias == 'hg':
                 # patch and reset hooks section of UI config to not run any
                 # hooks on fetching archives with subrepos
                 for k, v in c.rhodecode_repo._repo.ui.configitems('hooks'):
                     c.rhodecode_repo._repo.ui.setconfig('hooks', k, None)
 
             cs = c.rhodecode_repo.get_changeset(revision)
             content_type = settings.ARCHIVE_SPECS[fileformat][0]
         except ChangesetDoesNotExistError:
             return _('Unknown revision %s') % revision
         except EmptyRepositoryError:
             return _('Empty repository')
         except (ImproperArchiveTypeError, KeyError):
             return _('Unknown archive type')
 
         response.content_type = content_type
         response.content_disposition = 'attachment; filename=%s-%s%s' \
             % (repo_name, revision, ext)
 
         import tempfile
         archive = tempfile.mkstemp()[1]
         t = open(archive, 'wb')
         cs.fill_archive(stream=t, kind=fileformat, subrepos=subrepos)
 
         def get_chunked_archive(archive):
             stream = open(archive, 'rb')
             while True:
                 data = stream.read(4096)
                 if not data:
                     os.remove(archive)
                     break
                 yield data
 
         return get_chunked_archive(archive)
 
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def diff(self, repo_name, f_path):
         ignore_whitespace = request.GET.get('ignorews') == '1'
         line_context = request.GET.get('context', 3)
         diff1 = request.GET.get('diff1', '')
         diff2 = request.GET.get('diff2', '')
         c.action = request.GET.get('diff')
         c.no_changes = diff1 == diff2
         c.f_path = f_path
         c.big_diff = False
         c.anchor_url = anchor_url
         c.ignorews_url = _ignorews_url
         c.context_url = _context_url
         c.changes = OrderedDict()
         c.changes[diff2] = []
         try:
             if diff1 not in ['', None, 'None', '0' * 12, '0' * 40]:
                 c.changeset_1 = c.rhodecode_repo.get_changeset(diff1)
                 node1 = c.changeset_1.get_node(f_path)
             else:
                 c.changeset_1 = EmptyChangeset(repo=c.rhodecode_repo)
                 node1 = FileNode('.', '', changeset=c.changeset_1)
 
             if diff2 not in ['', None, 'None', '0' * 12, '0' * 40]:
                 c.changeset_2 = c.rhodecode_repo.get_changeset(diff2)
                 node2 = c.changeset_2.get_node(f_path)
             else:
                 c.changeset_2 = EmptyChangeset(repo=c.rhodecode_repo)
                 node2 = FileNode('.', '', changeset=c.changeset_2)
         except RepositoryError:
             return redirect(url('files_home', repo_name=c.repo_name,
                                 f_path=f_path))
 
         if c.action == 'download':
             _diff = diffs.get_gitdiff(node1, node2,
                                       ignore_whitespace=ignore_whitespace,
                                       context=line_context)
             diff = diffs.DiffProcessor(_diff, format='gitdiff')
 
             diff_name = '%s_vs_%s.diff' % (diff1, diff2)
             response.content_type = 'text/plain'
             response.content_disposition = 'attachment; filename=%s' \
                                                     % diff_name
             return diff.raw_diff()
 
         elif c.action == 'raw':
             _diff = diffs.get_gitdiff(node1, node2,
                                       ignore_whitespace=ignore_whitespace,
                                       context=line_context)
             diff = diffs.DiffProcessor(_diff, format='gitdiff')
             response.content_type = 'text/plain'
             return diff.raw_diff()
 
         else:
             fid = h.FID(diff2, node2.path)
             line_context_lcl = get_line_ctx(fid, request.GET)
             ign_whitespace_lcl = get_ignore_ws(fid, request.GET)
 
             lim = request.GET.get('fulldiff') or self.cut_off_limit
             _, cs1, cs2, diff, st = wrapped_diff(filenode_old=node1,
                                          filenode_new=node2,
                                          cut_off_limit=lim,
                                          ignore_whitespace=ign_whitespace_lcl,
                                          line_context=line_context_lcl,
                                          enable_comments=False)
 
             c.changes = [('', node2, diff, cs1, cs2, st,)]
 
         return render('files/file_diff.html')
 
     def _get_node_history(self, cs, f_path):
         changesets = cs.get_file_history(f_path)
         hist_l = []
 
         changesets_group = ([], _("Changesets"))
         branches_group = ([], _("Branches"))
         tags_group = ([], _("Tags"))
 
         for chs in changesets:
             n_desc = 'r%s:%s (%s)' % (chs.revision, chs.short_id, chs.branch)
             changesets_group[0].append((chs.raw_id, n_desc,))
 
         hist_l.append(changesets_group)
 
         for name, chs in c.rhodecode_repo.branches.items():
             branches_group[0].append((chs, name),)
         hist_l.append(branches_group)
 
         for name, chs in c.rhodecode_repo.tags.items():
             tags_group[0].append((chs, name),)
         hist_l.append(tags_group)
 
         return hist_l
 
     @jsonify
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def nodelist(self, repo_name, revision, f_path):
         if request.environ.get('HTTP_X_PARTIAL_XHR'):
             cs = self.__get_cs_or_redirect(revision, repo_name)
             _d, _f = ScmModel().get_nodes(repo_name, cs.raw_id, f_path,
                                           flat=False)
             return _d + _f

rhodecode/controllers/summary.py

@@ -1,233 +1,233 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.summary
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Summary controller for Rhodecode
 
     :created_on: Apr 18, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import traceback
 import calendar
 import logging
 from time import mktime
 from datetime import timedelta, date
 from itertools import product
 from urlparse import urlparse
 
-from vcs.exceptions import ChangesetError, EmptyRepositoryError, \
+from rhodecode.lib.vcs.exceptions import ChangesetError, EmptyRepositoryError, \
     NodeDoesNotExistError
 
 from pylons import tmpl_context as c, request, url, config
 from pylons.i18n.translation import _
 
 from beaker.cache import cache_region, region_invalidate
 
 from rhodecode.model.db import Statistics, CacheInvalidation
 from rhodecode.lib import ALL_READMES, ALL_EXTS
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.lib.utils import EmptyChangeset
 from rhodecode.lib.markup_renderer import MarkupRenderer
 from rhodecode.lib.celerylib import run_task
 from rhodecode.lib.celerylib.tasks import get_commits_stats, \
     LANGUAGES_EXTENSIONS_MAP
 from rhodecode.lib.helpers import RepoPage
 from rhodecode.lib.compat import json, OrderedDict
 
 log = logging.getLogger(__name__)
 
 README_FILES = [''.join([x[0][0], x[1][0]]) for x in
                     sorted(list(product(ALL_READMES, ALL_EXTS)),
                            key=lambda y:y[0][1] + y[1][1])]
 
 
 class SummaryController(BaseRepoController):
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def __before__(self):
         super(SummaryController, self).__before__()
 
     def index(self, repo_name):
         c.dbrepo = dbrepo = c.rhodecode_db_repo
         c.following = self.scm_model.is_following_repo(repo_name,
                                                 self.rhodecode_user.user_id)
 
         def url_generator(**kw):
             return url('shortlog_home', repo_name=repo_name, size=10, **kw)
 
         c.repo_changesets = RepoPage(c.rhodecode_repo, page=1,
                                      items_per_page=10, url=url_generator)
 
         if self.rhodecode_user.username == 'default':
             # for default(anonymous) user we don't need to pass credentials
             username = ''
             password = ''
         else:
             username = str(self.rhodecode_user.username)
             password = '@'
 
         parsed_url = urlparse(url.current(qualified=True))
 
         default_clone_uri = '{scheme}://{user}{pass}{netloc}{path}'
 
         uri_tmpl = config.get('clone_uri', default_clone_uri)
         uri_tmpl = uri_tmpl.replace('{', '%(').replace('}', ')s')
 
         uri_dict = {
            'user': username,
            'pass': password,
            'scheme': parsed_url.scheme,
            'netloc': parsed_url.netloc,
            'path': parsed_url.path
         }
         uri = uri_tmpl % uri_dict
         # generate another clone url by id
         uri_dict.update({'path': '/_%s' % c.dbrepo.repo_id})
         uri_id = uri_tmpl % uri_dict
 
         c.clone_repo_url = uri
         c.clone_repo_url_id = uri_id
         c.repo_tags = OrderedDict()
         for name, hash in c.rhodecode_repo.tags.items()[:10]:
             try:
                 c.repo_tags[name] = c.rhodecode_repo.get_changeset(hash)
             except ChangesetError:
                 c.repo_tags[name] = EmptyChangeset(hash)
 
         c.repo_branches = OrderedDict()
         for name, hash in c.rhodecode_repo.branches.items()[:10]:
             try:
                 c.repo_branches[name] = c.rhodecode_repo.get_changeset(hash)
             except ChangesetError:
                 c.repo_branches[name] = EmptyChangeset(hash)
 
         td = date.today() + timedelta(days=1)
         td_1m = td - timedelta(days=calendar.mdays[td.month])
         td_1y = td - timedelta(days=365)
 
         ts_min_m = mktime(td_1m.timetuple())
         ts_min_y = mktime(td_1y.timetuple())
         ts_max_y = mktime(td.timetuple())
 
         if dbrepo.enable_statistics:
             c.show_stats = True
             c.no_data_msg = _('No data loaded yet')
             run_task(get_commits_stats, c.dbrepo.repo_name, ts_min_y, ts_max_y)
         else:
             c.show_stats = False
             c.no_data_msg = _('Statistics are disabled for this repository')
         c.ts_min = ts_min_m
         c.ts_max = ts_max_y
 
         stats = self.sa.query(Statistics)\
             .filter(Statistics.repository == dbrepo)\
             .scalar()
 
         c.stats_percentage = 0
 
         if stats and stats.languages:
             c.no_data = False is dbrepo.enable_statistics
             lang_stats_d = json.loads(stats.languages)
             c.commit_data = stats.commit_activity
             c.overview_data = stats.commit_activity_combined
 
             lang_stats = ((x, {"count": y,
                                "desc": LANGUAGES_EXTENSIONS_MAP.get(x)})
                           for x, y in lang_stats_d.items())
 
             c.trending_languages = json.dumps(
                 sorted(lang_stats, reverse=True, key=lambda k: k[1])[:10]
             )
             last_rev = stats.stat_on_revision + 1
             c.repo_last_rev = c.rhodecode_repo.count()\
                 if c.rhodecode_repo.revisions else 0
             if last_rev == 0 or c.repo_last_rev == 0:
                 pass
             else:
                 c.stats_percentage = '%.2f' % ((float((last_rev)) /
                                                 c.repo_last_rev) * 100)
         else:
             c.commit_data = json.dumps({})
             c.overview_data = json.dumps([[ts_min_y, 0], [ts_max_y, 10]])
             c.trending_languages = json.dumps({})
             c.no_data = True
 
         c.enable_downloads = dbrepo.enable_downloads
         if c.enable_downloads:
             c.download_options = self._get_download_links(c.rhodecode_repo)
 
         c.readme_data, c.readme_file = self.__get_readme_data(c.rhodecode_repo)
         return render('summary/summary.html')
 
     def __get_readme_data(self, repo):
 
         @cache_region('long_term')
         def _get_readme_from_cache(key):
             readme_data = None
             readme_file = None
             log.debug('Fetching readme file')
             try:
                 cs = repo.get_changeset('tip')
                 renderer = MarkupRenderer()
                 for f in README_FILES:
                     try:
                         readme = cs.get_node(f)
                         readme_file = f
                         readme_data = renderer.render(readme.content, f)
                         log.debug('Found readme %s' % readme_file)
                         break
                     except NodeDoesNotExistError:
                         continue
             except ChangesetError:
                 pass
             except EmptyRepositoryError:
                 pass
             except Exception:
                 log.error(traceback.format_exc())
 
             return readme_data, readme_file
 
         key = repo.name + '_README'
         inv = CacheInvalidation.invalidate(key)
         if inv is not None:
             region_invalidate(_get_readme_from_cache, None, key)
             CacheInvalidation.set_valid(inv.cache_key)
         return _get_readme_from_cache(key)
 
     def _get_download_links(self, repo):
 
         download_l = []
 
         branches_group = ([], _("Branches"))
         tags_group = ([], _("Tags"))
 
         for name, chs in c.rhodecode_repo.branches.items():
             #chs = chs.split(':')[-1]
             branches_group[0].append((chs, name),)
         download_l.append(branches_group)
 
         for name, chs in c.rhodecode_repo.tags.items():
             #chs = chs.split(':')[-1]
             tags_group[0].append((chs, name),)
         download_l.append(tags_group)
 
         return download_l

rhodecode/lib/__init__.py

@@ -1,454 +1,454 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.__init__
     ~~~~~~~~~~~~~~~~~~~~~~~
 
     Some simple helper functions
 
     :created_on: Jan 5, 2011
     :author: marcink
     :copyright: (C) 2011-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import re
-from vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
 
 
 def __get_lem():
     from pygments import lexers
     from string import lower
     from collections import defaultdict
 
     d = defaultdict(lambda: [])
 
     def __clean(s):
         s = s.lstrip('*')
         s = s.lstrip('.')
 
         if s.find('[') != -1:
             exts = []
             start, stop = s.find('['), s.find(']')
 
             for suffix in s[start + 1:stop]:
                 exts.append(s[:s.find('[')] + suffix)
             return map(lower, exts)
         else:
             return map(lower, [s])
 
     for lx, t in sorted(lexers.LEXERS.items()):
         m = map(__clean, t[-2])
         if m:
             m = reduce(lambda x, y: x + y, m)
             for ext in m:
                 desc = lx.replace('Lexer', '')
                 d[ext].append(desc)
 
     return dict(d)
 
 # language map is also used by whoosh indexer, which for those specified
 # extensions will index it's content
 LANGUAGES_EXTENSIONS_MAP = __get_lem()
 
 # Additional mappings that are not present in the pygments lexers
 # NOTE: that this will overide any mappings in LANGUAGES_EXTENSIONS_MAP
 ADDITIONAL_MAPPINGS = {'xaml': 'XAML'}
 
 LANGUAGES_EXTENSIONS_MAP.update(ADDITIONAL_MAPPINGS)
 
 # list of readme files to search in file tree and display in summary
 # attached weights defines the search  order lower is first
 ALL_READMES = [
     ('readme', 0), ('README', 0), ('Readme', 0),
     ('doc/readme', 1), ('doc/README', 1), ('doc/Readme', 1),
     ('Docs/readme', 2), ('Docs/README', 2), ('Docs/Readme', 2),
     ('DOCS/readme', 2), ('DOCS/README', 2), ('DOCS/Readme', 2),
     ('docs/readme', 2), ('docs/README', 2), ('docs/Readme', 2),
 ]
 
 # extension together with weights to search lower is first
 RST_EXTS = [
     ('', 0), ('.rst', 1), ('.rest', 1),
-    ('.RST', 2) , ('.REST', 2),
+    ('.RST', 2), ('.REST', 2),
     ('.txt', 3), ('.TXT', 3)
 ]
 
 MARKDOWN_EXTS = [
     ('.md', 1), ('.MD', 1),
     ('.mkdn', 2), ('.MKDN', 2),
     ('.mdown', 3), ('.MDOWN', 3),
     ('.markdown', 4), ('.MARKDOWN', 4)
 ]
 
 PLAIN_EXTS = [('.text', 2), ('.TEXT', 2)]
 
 ALL_EXTS = MARKDOWN_EXTS + RST_EXTS + PLAIN_EXTS
 
 
 def str2bool(_str):
     """
     returs True/False value from given string, it tries to translate the
     string into boolean
 
     :param _str: string value to translate into boolean
     :rtype: boolean
     :returns: boolean from given string
     """
     if _str is None:
         return False
     if _str in (True, False):
         return _str
     _str = str(_str).strip().lower()
     return _str in ('t', 'true', 'y', 'yes', 'on', '1')
 
 
 def convert_line_endings(line, mode):
     """
     Converts a given line  "line end" accordingly to given mode
 
     Available modes are::
         0 - Unix
         1 - Mac
         2 - DOS
 
     :param line: given line to convert
     :param mode: mode to convert to
     :rtype: str
     :return: converted line according to mode
     """
     from string import replace
 
     if mode == 0:
             line = replace(line, '\r\n', '\n')
             line = replace(line, '\r', '\n')
     elif mode == 1:
             line = replace(line, '\r\n', '\r')
             line = replace(line, '\n', '\r')
     elif mode == 2:
-            import re
             line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line)
     return line
 
 
 def detect_mode(line, default):
     """
     Detects line break for given line, if line break couldn't be found
     given default value is returned
 
     :param line: str line
     :param default: default
     :rtype: int
     :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS
     """
     if line.endswith('\r\n'):
         return 2
     elif line.endswith('\n'):
         return 0
     elif line.endswith('\r'):
         return 1
     else:
         return default
 
 
 def generate_api_key(username, salt=None):
     """
     Generates unique API key for given username, if salt is not given
     it'll be generated from some random string
 
     :param username: username as string
     :param salt: salt to hash generate KEY
     :rtype: str
     :returns: sha1 hash from username+salt
     """
     from tempfile import _RandomNameSequence
     import hashlib
 
     if salt is None:
         salt = _RandomNameSequence().next()
 
     return hashlib.sha1(username + salt).hexdigest()
 
 
 def safe_unicode(str_, from_encoding='utf8'):
     """
     safe unicode function. Does few trick to turn str_ into unicode
 
     In case of UnicodeDecode error we try to return it with encoding detected
     by chardet library if it fails fallback to unicode with errors replaced
 
     :param str_: string to decode
     :rtype: unicode
     :returns: unicode object
     """
     if isinstance(str_, unicode):
         return str_
 
     try:
         return unicode(str_)
     except UnicodeDecodeError:
         pass
 
     try:
         return unicode(str_, from_encoding)
     except UnicodeDecodeError:
         pass
 
     try:
         import chardet
         encoding = chardet.detect(str_)['encoding']
         if encoding is None:
             raise Exception()
         return str_.decode(encoding)
     except (ImportError, UnicodeDecodeError, Exception):
         return unicode(str_, from_encoding, 'replace')
 
 
 def safe_str(unicode_, to_encoding='utf8'):
     """
     safe str function. Does few trick to turn unicode_ into string
 
     In case of UnicodeEncodeError we try to return it with encoding detected
     by chardet library if it fails fallback to string with errors replaced
 
     :param unicode_: unicode to encode
     :rtype: str
     :returns: str object
     """
 
     if not isinstance(unicode_, basestring):
         return str(unicode_)
 
     if isinstance(unicode_, str):
         return unicode_
 
     try:
         return unicode_.encode(to_encoding)
     except UnicodeEncodeError:
         pass
 
     try:
         import chardet
         encoding = chardet.detect(unicode_)['encoding']
         print encoding
         if encoding is None:
             raise UnicodeEncodeError()
 
         return unicode_.encode(encoding)
     except (ImportError, UnicodeEncodeError):
         return unicode_.encode(to_encoding, 'replace')
 
     return safe_str
 
 
 def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs):
     """
     Custom engine_from_config functions that makes sure we use NullPool for
     file based sqlite databases. This prevents errors on sqlite. This only
     applies to sqlalchemy versions < 0.7.0
 
     """
     import sqlalchemy
     from sqlalchemy import engine_from_config as efc
     import logging
 
     if int(sqlalchemy.__version__.split('.')[1]) < 7:
 
         # This solution should work for sqlalchemy < 0.7.0, and should use
         # proxy=TimerProxy() for execution time profiling
 
         from sqlalchemy.pool import NullPool
         url = configuration[prefix + 'url']
 
         if url.startswith('sqlite'):
             kwargs.update({'poolclass': NullPool})
         return efc(configuration, prefix, **kwargs)
     else:
         import time
         from sqlalchemy import event
         from sqlalchemy.engine import Engine
 
         log = logging.getLogger('sqlalchemy.engine')
         BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = xrange(30, 38)
         engine = efc(configuration, prefix, **kwargs)
 
         def color_sql(sql):
             COLOR_SEQ = "\033[1;%dm"
             COLOR_SQL = YELLOW
             normal = '\x1b[0m'
             return ''.join([COLOR_SEQ % COLOR_SQL, sql, normal])
 
         if configuration['debug']:
             #attach events only for debug configuration
 
             def before_cursor_execute(conn, cursor, statement,
                                     parameters, context, executemany):
                 context._query_start_time = time.time()
                 log.info(color_sql(">>>>> STARTING QUERY >>>>>"))
 
 
             def after_cursor_execute(conn, cursor, statement,
                                     parameters, context, executemany):
                 total = time.time() - context._query_start_time
                 log.info(color_sql("<<<<< TOTAL TIME: %f <<<<<" % total))
 
             event.listen(engine, "before_cursor_execute",
                          before_cursor_execute)
             event.listen(engine, "after_cursor_execute",
                          after_cursor_execute)
 
     return engine
 
 
 def age(curdate):
     """
     turns a datetime into an age string.
 
     :param curdate: datetime object
     :rtype: unicode
     :returns: unicode words describing age
     """
 
     from datetime import datetime
     from webhelpers.date import time_ago_in_words
 
-    _ = lambda s:s
+    _ = lambda s: s
 
     if not curdate:
         return ''
 
     agescales = [(_(u"year"), 3600 * 24 * 365),
                  (_(u"month"), 3600 * 24 * 30),
                  (_(u"day"), 3600 * 24),
                  (_(u"hour"), 3600),
                  (_(u"minute"), 60),
                  (_(u"second"), 1), ]
 
     age = datetime.now() - curdate
     age_seconds = (age.days * agescales[2][1]) + age.seconds
     pos = 1
     for scale in agescales:
         if scale[1] <= age_seconds:
-            if pos == 6:pos = 5
+            if pos == 6:
+                pos = 5
             return '%s %s' % (time_ago_in_words(curdate,
                                                 agescales[pos][0]), _('ago'))
         pos += 1
 
     return _(u'just now')
 
 
 def uri_filter(uri):
     """
     Removes user:password from given url string
 
     :param uri:
     :rtype: unicode
     :returns: filtered list of strings
     """
     if not uri:
         return ''
 
     proto = ''
 
     for pat in ('https://', 'http://'):
         if uri.startswith(pat):
             uri = uri[len(pat):]
             proto = pat
             break
 
     # remove passwords and username
     uri = uri[uri.find('@') + 1:]
 
     # get the port
     cred_pos = uri.find(':')
     if cred_pos == -1:
         host, port = uri, None
     else:
         host, port = uri[:cred_pos], uri[cred_pos + 1:]
 
     return filter(None, [proto, host, port])
 
 
 def credentials_filter(uri):
     """
     Returns a url with removed credentials
 
     :param uri:
     """
 
     uri = uri_filter(uri)
     #check if we have port
     if len(uri) > 2 and uri[2]:
         uri[2] = ':' + uri[2]
 
     return ''.join(uri)
 
 
 def get_changeset_safe(repo, rev):
     """
     Safe version of get_changeset if this changeset doesn't exists for a
     repo it returns a Dummy one instead
 
     :param repo:
     :param rev:
     """
-    from vcs.backends.base import BaseRepository
-    from vcs.exceptions import RepositoryError
+    from rhodecode.lib.vcs.backends.base import BaseRepository
+    from rhodecode.lib.vcs.exceptions import RepositoryError
     if not isinstance(repo, BaseRepository):
         raise Exception('You must pass an Repository '
                         'object as first argument got %s', type(repo))
 
     try:
         cs = repo.get_changeset(rev)
     except RepositoryError:
         from rhodecode.lib.utils import EmptyChangeset
         cs = EmptyChangeset(requested_revision=rev)
     return cs
 
 
 def get_current_revision(quiet=False):
     """
     Returns tuple of (number, id) from repository containing this package
     or None if repository could not be found.
 
     :param quiet: prints error for fetching revision if True
     """
 
     try:
-        from vcs import get_repo
-        from vcs.utils.helpers import get_scm
+        from rhodecode.lib.vcs import get_repo
+        from rhodecode.lib.vcs.utils.helpers import get_scm
         repopath = os.path.join(os.path.dirname(__file__), '..', '..')
         scm = get_scm(repopath)[0]
         repo = get_repo(path=repopath, alias=scm)
         tip = repo.get_changeset()
         return (tip.revision, tip.short_id)
     except Exception, err:
         if not quiet:
             print ("Cannot retrieve rhodecode's revision. Original error "
                    "was: %s" % err)
         return None
 
 
 def extract_mentioned_users(s):
     """
     Returns unique usernames from given string s that have @mention
 
     :param s: string to get mentions
     """
     usrs = {}
     for username in re.findall(r'(?:^@|\s@)(\w+)', s):
         usrs[username] = username
 
     return sorted(usrs.keys())

rhodecode/lib/annotate.py

@@ -1,190 +1,190 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.annotate
     ~~~~~~~~~~~~~~~~~~~~~~
 
     Anontation library for usage in rhodecode, previously part of vcs
 
     :created_on: Dec 4, 2011
     :author: marcink
     :copyright: (C) 2011-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 
-from vcs.exceptions import VCSError
-from vcs.nodes import FileNode
+from rhodecode.lib.vcs.exceptions import VCSError
+from rhodecode.lib.vcs.nodes import FileNode
 from pygments.formatters import HtmlFormatter
 from pygments import highlight
 
 import StringIO
 
 
 def annotate_highlight(filenode, annotate_from_changeset_func=None,
         order=None, headers=None, **options):
     """
     Returns html portion containing annotated table with 3 columns: line
     numbers, changeset information and pygmentized line of code.
 
     :param filenode: FileNode object
     :param annotate_from_changeset_func: function taking changeset and
       returning single annotate cell; needs break line at the end
     :param order: ordered sequence of ``ls`` (line numbers column),
       ``annotate`` (annotate column), ``code`` (code column); Default is
       ``['ls', 'annotate', 'code']``
     :param headers: dictionary with headers (keys are whats in ``order``
       parameter)
     """
     options['linenos'] = True
     formatter = AnnotateHtmlFormatter(filenode=filenode, order=order,
         headers=headers,
         annotate_from_changeset_func=annotate_from_changeset_func, **options)
     lexer = filenode.lexer
     highlighted = highlight(filenode.content, lexer, formatter)
     return highlighted
 
 
 class AnnotateHtmlFormatter(HtmlFormatter):
 
     def __init__(self, filenode, annotate_from_changeset_func=None,
             order=None, **options):
         """
         If ``annotate_from_changeset_func`` is passed it should be a function
         which returns string from the given changeset. For example, we may pass
         following function as ``annotate_from_changeset_func``::
 
             def changeset_to_anchor(changeset):
                 return '<a href="/changesets/%s/">%s</a>\n' %\
                        (changeset.id, changeset.id)
 
         :param annotate_from_changeset_func: see above
         :param order: (default: ``['ls', 'annotate', 'code']``); order of
           columns;
         :param options: standard pygment's HtmlFormatter options, there is
           extra option tough, ``headers``. For instance we can pass::
 
              formatter = AnnotateHtmlFormatter(filenode, headers={
                 'ls': '#',
                 'annotate': 'Annotate',
                 'code': 'Code',
              })
 
         """
         super(AnnotateHtmlFormatter, self).__init__(**options)
         self.annotate_from_changeset_func = annotate_from_changeset_func
         self.order = order or ('ls', 'annotate', 'code')
         headers = options.pop('headers', None)
         if headers and not ('ls' in headers and 'annotate' in headers and
             'code' in headers):
             raise ValueError("If headers option dict is specified it must "
                 "all 'ls', 'annotate' and 'code' keys")
         self.headers = headers
         if isinstance(filenode, FileNode):
             self.filenode = filenode
         else:
             raise VCSError("This formatter expect FileNode parameter, not %r"
                 % type(filenode))
 
     def annotate_from_changeset(self, changeset):
         """
         Returns full html line for single changeset per annotated line.
         """
         if self.annotate_from_changeset_func:
             return self.annotate_from_changeset_func(changeset)
         else:
             return ''.join((changeset.id, '\n'))
 
     def _wrap_tablelinenos(self, inner):
         dummyoutfile = StringIO.StringIO()
         lncount = 0
         for t, line in inner:
             if t:
                 lncount += 1
             dummyoutfile.write(line)
 
         fl = self.linenostart
         mw = len(str(lncount + fl - 1))
         sp = self.linenospecial
         st = self.linenostep
         la = self.lineanchors
         aln = self.anchorlinenos
         if sp:
             lines = []
 
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if i % sp == 0:
                         if aln:
                             lines.append('<a href="#%s-%d" class="special">'
                                          '%*d</a>' %
                                          (la, i, mw, i))
                         else:
                             lines.append('<span class="special">'
                                          '%*d</span>' % (mw, i))
                     else:
                         if aln:
                             lines.append('<a href="#%s-%d">'
                                          '%*d</a>' % (la, i, mw, i))
                         else:
                             lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
         else:
             lines = []
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if aln:
                         lines.append('<a href="#%s-%d">%*d</a>' \
                                      % (la, i, mw, i))
                     else:
                         lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
 
         annotate_changesets = [tup[1] for tup in self.filenode.annotate]
         # If pygments cropped last lines break we need do that too
         ln_cs = len(annotate_changesets)
         ln_ = len(ls.splitlines())
         if  ln_cs > ln_:
             annotate_changesets = annotate_changesets[:ln_ - ln_cs]
         annotate = ''.join((self.annotate_from_changeset(changeset)
             for changeset in annotate_changesets))
         # in case you wonder about the seemingly redundant <div> here:
         # since the content in the other cell also is wrapped in a div,
         # some browsers in some configurations seem to mess up the formatting.
         '''
         yield 0, ('<table class="%stable">' % self.cssclass +
                   '<tr><td class="linenos"><div class="linenodiv"><pre>' +
                   ls + '</pre></div></td>' +
                   '<td class="code">')
         yield 0, dummyoutfile.getvalue()
         yield 0, '</td></tr></table>'
 
         '''
         headers_row = []
         if self.headers:
             headers_row = ['<tr class="annotate-header">']
             for key in self.order:
                 td = ''.join(('<td>', self.headers[key], '</td>'))
                 headers_row.append(td)
             headers_row.append('</tr>')
 
         body_row_start = ['<tr>']
         for key in self.order:
             if key == 'ls':
                 body_row_start.append(
                     '<td class="linenos"><div class="linenodiv"><pre>' +
                     ls + '</pre></div></td>')
             elif key == 'annotate':
                 body_row_start.append(
                     '<td class="annotate"><div class="annotatediv"><pre>' +
                     annotate + '</pre></div></td>')
             elif key == 'code':
                 body_row_start.append('<td class="code">')
         yield 0, ('<table class="%stable">' % self.cssclass +
                   ''.join(headers_row) +
                   ''.join(body_row_start)
                   )
         yield 0, dummyoutfile.getvalue()
         yield 0, '</td></tr></table>'

rhodecode/lib/celerylib/__init__.py

@@ -1,128 +1,128 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.celerylib.__init__
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     celery libs for RhodeCode
 
     :created_on: Nov 27, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import sys
 import socket
 import traceback
 import logging
 from os.path import dirname as dn, join as jn
 from pylons import config
 
 from hashlib import md5
 from decorator import decorator
 
-from vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
 from rhodecode import CELERY_ON
 from rhodecode.lib import str2bool, safe_str
 from rhodecode.lib.pidlock import DaemonLock, LockHeld
 from rhodecode.model import init_model
 from rhodecode.model import meta
 from rhodecode.model.db import Statistics, Repository, User
 
 from sqlalchemy import engine_from_config
 
 from celery.messaging import establish_connection
 
 log = logging.getLogger(__name__)
 
 
 class ResultWrapper(object):
     def __init__(self, task):
         self.task = task
 
     @LazyProperty
     def result(self):
         return self.task
 
 
 def run_task(task, *args, **kwargs):
     if CELERY_ON:
         try:
             t = task.apply_async(args=args, kwargs=kwargs)
             log.info('running task %s:%s' % (t.task_id, task))
             return t
 
         except socket.error, e:
             if isinstance(e, IOError) and e.errno == 111:
                 log.debug('Unable to connect to celeryd. Sync execution')
             else:
                 log.error(traceback.format_exc())
         except KeyError, e:
                 log.debug('Unable to connect to celeryd. Sync execution')
         except Exception, e:
             log.error(traceback.format_exc())
 
     log.debug('executing task %s in sync mode' % task)
     return ResultWrapper(task(*args, **kwargs))
 
 
 def __get_lockkey(func, *fargs, **fkwargs):
     params = list(fargs)
     params.extend(['%s-%s' % ar for ar in fkwargs.items()])
 
     func_name = str(func.__name__) if hasattr(func, '__name__') else str(func)
 
     lockkey = 'task_%s.lock' % \
         md5(func_name + '-' + '-'.join(map(safe_str, params))).hexdigest()
     return lockkey
 
 
 def locked_task(func):
     def __wrapper(func, *fargs, **fkwargs):
         lockkey = __get_lockkey(func, *fargs, **fkwargs)
         lockkey_path = config['here']
 
         log.info('running task with lockkey %s' % lockkey)
         try:
             l = DaemonLock(file_=jn(lockkey_path, lockkey))
             ret = func(*fargs, **fkwargs)
             l.release()
             return ret
         except LockHeld:
             log.info('LockHeld')
             return 'Task with key %s already running' % lockkey
 
     return decorator(__wrapper, func)
 
 
 def get_session():
     if CELERY_ON:
         engine = engine_from_config(config, 'sqlalchemy.db1.')
         init_model(engine)
     sa = meta.Session
     return sa
 
 
 def dbsession(func):
     def __wrapper(func, *fargs, **fkwargs):
         try:
             ret = func(*fargs, **fkwargs)
             return ret
         finally:
             if CELERY_ON:
                 meta.Session.remove()
 
     return decorator(__wrapper, func)

rhodecode/lib/celerylib/tasks.py

@@ -1,414 +1,414 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.celerylib.tasks
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     RhodeCode task modules, containing all task that suppose to be run
     by celery daemon
 
     :created_on: Oct 6, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 from celery.decorators import task
 
 import os
 import traceback
 import logging
 from os.path import join as jn
 
 from time import mktime
 from operator import itemgetter
 from string import lower
 
 from pylons import config, url
 from pylons.i18n.translation import _
 
-from vcs import get_backend
+from rhodecode.lib.vcs import get_backend
 
 from rhodecode import CELERY_ON
 from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP, safe_str
 from rhodecode.lib.celerylib import run_task, locked_task, dbsession, \
     str2bool, __get_lockkey, LockHeld, DaemonLock, get_session
 from rhodecode.lib.helpers import person
 from rhodecode.lib.rcmail.smtp_mailer import SmtpMailer
 from rhodecode.lib.utils import add_cache, action_logger
 from rhodecode.lib.compat import json, OrderedDict
 
 from rhodecode.model.db import Statistics, Repository, User
 
 
 add_cache(config)
 
 __all__ = ['whoosh_index', 'get_commits_stats',
            'reset_user_password', 'send_email']
 
 
 def get_logger(cls):
     if CELERY_ON:
         try:
             log = cls.get_logger()
         except:
             log = logging.getLogger(__name__)
     else:
         log = logging.getLogger(__name__)
 
     return log
 
 
 @task(ignore_result=True)
 @locked_task
 @dbsession
 def whoosh_index(repo_location, full_index):
     from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
     log = whoosh_index.get_logger(whoosh_index)
     DBS = get_session()
 
     index_location = config['index_dir']
     WhooshIndexingDaemon(index_location=index_location,
                          repo_location=repo_location, sa=DBS)\
                          .run(full_index=full_index)
 
 
 @task(ignore_result=True)
 @dbsession
 def get_commits_stats(repo_name, ts_min_y, ts_max_y):
     log = get_logger(get_commits_stats)
     DBS = get_session()
     lockkey = __get_lockkey('get_commits_stats', repo_name, ts_min_y,
                             ts_max_y)
     lockkey_path = config['here']
 
     log.info('running task with lockkey %s' % lockkey)
 
     try:
         lock = l = DaemonLock(file_=jn(lockkey_path, lockkey))
 
         # for js data compatibility cleans the key for person from '
         akc = lambda k: person(k).replace('"', "")
 
         co_day_auth_aggr = {}
         commits_by_day_aggregate = {}
         repo = Repository.get_by_repo_name(repo_name)
         if repo is None:
             return True
 
         repo = repo.scm_instance
         repo_size = repo.count()
         # return if repo have no revisions
         if repo_size < 1:
             lock.release()
             return True
 
         skip_date_limit = True
         parse_limit = int(config['app_conf'].get('commit_parse_limit'))
         last_rev = None
         last_cs = None
         timegetter = itemgetter('time')
 
         dbrepo = DBS.query(Repository)\
             .filter(Repository.repo_name == repo_name).scalar()
         cur_stats = DBS.query(Statistics)\
             .filter(Statistics.repository == dbrepo).scalar()
 
         if cur_stats is not None:
             last_rev = cur_stats.stat_on_revision
 
         if last_rev == repo.get_changeset().revision and repo_size > 1:
             # pass silently without any work if we're not on first revision or
             # current state of parsing revision(from db marker) is the
             # last revision
             lock.release()
             return True
 
         if cur_stats:
             commits_by_day_aggregate = OrderedDict(json.loads(
                                         cur_stats.commit_activity_combined))
             co_day_auth_aggr = json.loads(cur_stats.commit_activity)
 
         log.debug('starting parsing %s' % parse_limit)
         lmktime = mktime
 
         last_rev = last_rev + 1 if last_rev >= 0 else 0
         log.debug('Getting revisions from %s to %s' % (
              last_rev, last_rev + parse_limit)
         )
         for cs in repo[last_rev:last_rev + parse_limit]:
             last_cs = cs  # remember last parsed changeset
             k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1],
                           cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0])
 
             if akc(cs.author) in co_day_auth_aggr:
                 try:
                     l = [timegetter(x) for x in
                          co_day_auth_aggr[akc(cs.author)]['data']]
                     time_pos = l.index(k)
                 except ValueError:
                     time_pos = False
 
                 if time_pos >= 0 and time_pos is not False:
 
                     datadict = \
                         co_day_auth_aggr[akc(cs.author)]['data'][time_pos]
 
                     datadict["commits"] += 1
                     datadict["added"] += len(cs.added)
                     datadict["changed"] += len(cs.changed)
                     datadict["removed"] += len(cs.removed)
 
                 else:
                     if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
 
                         datadict = {"time": k,
                                     "commits": 1,
                                     "added": len(cs.added),
                                     "changed": len(cs.changed),
                                     "removed": len(cs.removed),
                                    }
                         co_day_auth_aggr[akc(cs.author)]['data']\
                             .append(datadict)
 
             else:
                 if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
                     co_day_auth_aggr[akc(cs.author)] = {
                                         "label": akc(cs.author),
                                         "data": [{"time":k,
                                                  "commits":1,
                                                  "added":len(cs.added),
                                                  "changed":len(cs.changed),
                                                  "removed":len(cs.removed),
                                                  }],
                                         "schema": ["commits"],
                                         }
 
             #gather all data by day
             if k in commits_by_day_aggregate:
                 commits_by_day_aggregate[k] += 1
             else:
                 commits_by_day_aggregate[k] = 1
 
         overview_data = sorted(commits_by_day_aggregate.items(),
                                key=itemgetter(0))
 
         if not co_day_auth_aggr:
             co_day_auth_aggr[akc(repo.contact)] = {
                 "label": akc(repo.contact),
                 "data": [0, 1],
                 "schema": ["commits"],
             }
 
         stats = cur_stats if cur_stats else Statistics()
         stats.commit_activity = json.dumps(co_day_auth_aggr)
         stats.commit_activity_combined = json.dumps(overview_data)
 
         log.debug('last revison %s' % last_rev)
         leftovers = len(repo.revisions[last_rev:])
         log.debug('revisions to parse %s' % leftovers)
 
         if last_rev == 0 or leftovers < parse_limit:
             log.debug('getting code trending stats')
             stats.languages = json.dumps(__get_codes_stats(repo_name))
 
         try:
             stats.repository = dbrepo
             stats.stat_on_revision = last_cs.revision if last_cs else 0
             DBS.add(stats)
             DBS.commit()
         except:
             log.error(traceback.format_exc())
             DBS.rollback()
             lock.release()
             return False
 
         #final release
         lock.release()
 
         #execute another task if celery is enabled
         if len(repo.revisions) > 1 and CELERY_ON:
             run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y)
         return True
     except LockHeld:
         log.info('LockHeld')
         return 'Task with key %s already running' % lockkey
 
 @task(ignore_result=True)
 @dbsession
 def send_password_link(user_email):
     from rhodecode.model.notification import EmailNotificationModel
 
     log = get_logger(send_password_link)
     DBS = get_session()
 
     try:
         user = User.get_by_email(user_email)
         if user:
             log.debug('password reset user found %s' % user)
             link = url('reset_password_confirmation', key=user.api_key,
                        qualified=True)
             reg_type = EmailNotificationModel.TYPE_PASSWORD_RESET
             body = EmailNotificationModel().get_email_tmpl(reg_type,
                                                 **{'user':user.short_contact,
                                                    'reset_url':link})
             log.debug('sending email')
             run_task(send_email, user_email,
                      _("password reset link"), body)
             log.info('send new password mail to %s' % user_email)
         else:
             log.debug("password reset email %s not found" % user_email)
     except:
         log.error(traceback.format_exc())
         return False
 
     return True
 
 @task(ignore_result=True)
 @dbsession
 def reset_user_password(user_email):
     from rhodecode.lib import auth
 
     log = get_logger(reset_user_password)
     DBS = get_session()
 
     try:
         try:
             user = User.get_by_email(user_email)
             new_passwd = auth.PasswordGenerator().gen_password(8,
                              auth.PasswordGenerator.ALPHABETS_BIG_SMALL)
             if user:
                 user.password = auth.get_crypt_password(new_passwd)
                 user.api_key = auth.generate_api_key(user.username)
                 DBS.add(user)
                 DBS.commit()
                 log.info('change password for %s' % user_email)
             if new_passwd is None:
                 raise Exception('unable to generate new password')
         except:
             log.error(traceback.format_exc())
             DBS.rollback()
 
         run_task(send_email, user_email,
                  'Your new password',
                  'Your new RhodeCode password:%s' % (new_passwd))
         log.info('send new password mail to %s' % user_email)
 
     except:
         log.error('Failed to update user password')
         log.error(traceback.format_exc())
 
     return True
 
 
 @task(ignore_result=True)
 @dbsession
 def send_email(recipients, subject, body, html_body=''):
     """
     Sends an email with defined parameters from the .ini files.
 
     :param recipients: list of recipients, it this is empty the defined email
         address from field 'email_to' is used instead
     :param subject: subject of the mail
     :param body: body of the mail
     :param html_body: html version of body
     """
     log = get_logger(send_email)
     DBS = get_session()
 
     email_config = config
     subject = "%s %s" % (email_config.get('email_prefix'), subject)
     if not recipients:
         # if recipients are not defined we send to email_config + all admins
         admins = [u.email for u in User.query()
                   .filter(User.admin == True).all()]
         recipients = [email_config.get('email_to')] + admins
 
     mail_from = email_config.get('app_email_from', 'RhodeCode')
     user = email_config.get('smtp_username')
     passwd = email_config.get('smtp_password')
     mail_server = email_config.get('smtp_server')
     mail_port = email_config.get('smtp_port')
     tls = str2bool(email_config.get('smtp_use_tls'))
     ssl = str2bool(email_config.get('smtp_use_ssl'))
     debug = str2bool(config.get('debug'))
     smtp_auth = email_config.get('smtp_auth')
 
     try:
         m = SmtpMailer(mail_from, user, passwd, mail_server, smtp_auth,
                        mail_port, ssl, tls, debug=debug)
         m.send(recipients, subject, body, html_body)
     except:
         log.error('Mail sending failed')
         log.error(traceback.format_exc())
         return False
     return True
 
 
 @task(ignore_result=True)
 @dbsession
 def create_repo_fork(form_data, cur_user):
     """
     Creates a fork of repository using interval VCS methods
 
     :param form_data:
     :param cur_user:
     """
     from rhodecode.model.repo import RepoModel
 
     log = get_logger(create_repo_fork)
     DBS = get_session()
 
     base_path = Repository.base_path()
 
     RepoModel(DBS).create(form_data, cur_user, just_db=True, fork=True)
 
     alias = form_data['repo_type']
     org_repo_name = form_data['org_path']
     fork_name = form_data['repo_name_full']
     update_after_clone = form_data['update_after_clone']
     source_repo_path = os.path.join(base_path, org_repo_name)
     destination_fork_path = os.path.join(base_path, fork_name)
 
     log.info('creating fork of %s as %s', source_repo_path,
              destination_fork_path)
     backend = get_backend(alias)
     backend(safe_str(destination_fork_path), create=True,
             src_url=safe_str(source_repo_path),
             update_after_clone=update_after_clone)
     action_logger(cur_user, 'user_forked_repo:%s' % fork_name,
                    org_repo_name, '', DBS)
 
     action_logger(cur_user, 'user_created_fork:%s' % fork_name,
                    fork_name, '', DBS)
     # finally commit at latest possible stage
     DBS.commit()
 
 def __get_codes_stats(repo_name):
     repo = Repository.get_by_repo_name(repo_name).scm_instance
 
     tip = repo.get_changeset()
     code_stats = {}
 
     def aggregate(cs):
         for f in cs[2]:
             ext = lower(f.extension)
             if ext in LANGUAGES_EXTENSIONS_MAP.keys() and not f.is_binary:
                 if ext in code_stats:
                     code_stats[ext] += 1
                 else:
                     code_stats[ext] = 1
 
     map(aggregate, tip.walk('/'))
 
     return code_stats or {}

rhodecode/lib/dbmigrate/schema/db_1_2_0.py

@@ -1,1098 +1,1098 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.model.db
     ~~~~~~~~~~~~~~~~~~
 
     Database Models for RhodeCode
 
     :created_on: Apr 08, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import logging
 import datetime
 import traceback
 from datetime import date
 
 from sqlalchemy import *
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import relationship, joinedload, class_mapper, validates
 from beaker.cache import cache_region, region_invalidate
 
-from vcs import get_backend
-from vcs.utils.helpers import get_scm
-from vcs.exceptions import VCSError
-from vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs import get_backend
+from rhodecode.lib.vcs.utils.helpers import get_scm
+from rhodecode.lib.vcs.exceptions import VCSError
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
 
 from rhodecode.lib import str2bool, safe_str, get_changeset_safe, \
     generate_api_key, safe_unicode
 from rhodecode.lib.exceptions import UsersGroupsAssignedException
 from rhodecode.lib.compat import json
 
 from rhodecode.model.meta import Base, Session
 from rhodecode.lib.caching_query import FromCache
 
 
 log = logging.getLogger(__name__)
 
 #==============================================================================
 # BASE CLASSES
 #==============================================================================
 
 class ModelSerializer(json.JSONEncoder):
     """
     Simple Serializer for JSON,
 
     usage::
 
         to make object customized for serialization implement a __json__
         method that will return a dict for serialization into json
 
     example::
 
         class Task(object):
 
             def __init__(self, name, value):
                 self.name = name
                 self.value = value
 
             def __json__(self):
                 return dict(name=self.name,
                             value=self.value)
 
     """
 
     def default(self, obj):
 
         if hasattr(obj, '__json__'):
             return obj.__json__()
         else:
             return json.JSONEncoder.default(self, obj)
 
 class BaseModel(object):
     """Base Model for all classess
 
     """
 
     @classmethod
     def _get_keys(cls):
         """return column names for this model """
         return class_mapper(cls).c.keys()
 
     def get_dict(self):
         """return dict with keys and values corresponding
         to this model data """
 
         d = {}
         for k in self._get_keys():
             d[k] = getattr(self, k)
         return d
 
     def get_appstruct(self):
         """return list with keys and values tupples corresponding
         to this model data """
 
         l = []
         for k in self._get_keys():
             l.append((k, getattr(self, k),))
         return l
 
     def populate_obj(self, populate_dict):
         """populate model with data from given populate_dict"""
 
         for k in self._get_keys():
             if k in populate_dict:
                 setattr(self, k, populate_dict[k])
 
     @classmethod
     def query(cls):
         return Session.query(cls)
 
     @classmethod
     def get(cls, id_):
         if id_:
             return cls.query().get(id_)
 
     @classmethod
     def getAll(cls):
         return cls.query().all()
 
     @classmethod
     def delete(cls, id_):
         obj = cls.query().get(id_)
         Session.delete(obj)
         Session.commit()
 
 
 class RhodeCodeSetting(Base, BaseModel):
     __tablename__ = 'rhodecode_settings'
     __table_args__ = (UniqueConstraint('app_settings_name'), {'extend_existing':True})
     app_settings_id = Column("app_settings_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     app_settings_name = Column("app_settings_name", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     _app_settings_value = Column("app_settings_value", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
     def __init__(self, k='', v=''):
         self.app_settings_name = k
         self.app_settings_value = v
 
 
     @validates('_app_settings_value')
     def validate_settings_value(self, key, val):
         assert type(val) == unicode
         return val
 
     @hybrid_property
     def app_settings_value(self):
         v = self._app_settings_value
         if v == 'ldap_active':
             v = str2bool(v)
         return v
 
     @app_settings_value.setter
     def app_settings_value(self, val):
         """
         Setter that will always make sure we use unicode in app_settings_value
 
         :param val:
         """
         self._app_settings_value = safe_unicode(val)
 
     def __repr__(self):
         return "<%s('%s:%s')>" % (self.__class__.__name__,
                                   self.app_settings_name, self.app_settings_value)
 
 
     @classmethod
     def get_by_name(cls, ldap_key):
         return cls.query()\
             .filter(cls.app_settings_name == ldap_key).scalar()
 
     @classmethod
     def get_app_settings(cls, cache=False):
 
         ret = cls.query()
 
         if cache:
             ret = ret.options(FromCache("sql_cache_short", "get_hg_settings"))
 
         if not ret:
             raise Exception('Could not get application settings !')
         settings = {}
         for each in ret:
             settings['rhodecode_' + each.app_settings_name] = \
                 each.app_settings_value
 
         return settings
 
     @classmethod
     def get_ldap_settings(cls, cache=False):
         ret = cls.query()\
                 .filter(cls.app_settings_name.startswith('ldap_')).all()
         fd = {}
         for row in ret:
             fd.update({row.app_settings_name:row.app_settings_value})
 
         return fd
 
 
 class RhodeCodeUi(Base, BaseModel):
     __tablename__ = 'rhodecode_ui'
     __table_args__ = (UniqueConstraint('ui_key'), {'extend_existing':True})
 
     HOOK_UPDATE = 'changegroup.update'
     HOOK_REPO_SIZE = 'changegroup.repo_size'
     HOOK_PUSH = 'pretxnchangegroup.push_logger'
     HOOK_PULL = 'preoutgoing.pull_logger'
 
     ui_id = Column("ui_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     ui_section = Column("ui_section", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_key = Column("ui_key", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_value = Column("ui_value", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_active = Column("ui_active", Boolean(), nullable=True, unique=None, default=True)
 
 
     @classmethod
     def get_by_key(cls, key):
         return cls.query().filter(cls.ui_key == key)
 
 
     @classmethod
     def get_builtin_hooks(cls):
         q = cls.query()
         q = q.filter(cls.ui_key.in_([cls.HOOK_UPDATE,
                                     cls.HOOK_REPO_SIZE,
                                     cls.HOOK_PUSH, cls.HOOK_PULL]))
         return q.all()
 
     @classmethod
     def get_custom_hooks(cls):
         q = cls.query()
         q = q.filter(~cls.ui_key.in_([cls.HOOK_UPDATE,
                                     cls.HOOK_REPO_SIZE,
                                     cls.HOOK_PUSH, cls.HOOK_PULL]))
         q = q.filter(cls.ui_section == 'hooks')
         return q.all()
 
     @classmethod
     def create_or_update_hook(cls, key, val):
         new_ui = cls.get_by_key(key).scalar() or cls()
         new_ui.ui_section = 'hooks'
         new_ui.ui_active = True
         new_ui.ui_key = key
         new_ui.ui_value = val
 
         Session.add(new_ui)
         Session.commit()
 
 
 class User(Base, BaseModel):
     __tablename__ = 'users'
     __table_args__ = (UniqueConstraint('username'), UniqueConstraint('email'), {'extend_existing':True})
     user_id = Column("user_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     username = Column("username", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     password = Column("password", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     active = Column("active", Boolean(), nullable=True, unique=None, default=None)
     admin = Column("admin", Boolean(), nullable=True, unique=None, default=False)
     name = Column("name", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     lastname = Column("lastname", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     email = Column("email", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     last_login = Column("last_login", DateTime(timezone=False), nullable=True, unique=None, default=None)
     ldap_dn = Column("ldap_dn", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     api_key = Column("api_key", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
     user_log = relationship('UserLog', cascade='all')
     user_perms = relationship('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id", cascade='all')
 
     repositories = relationship('Repository')
     user_followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_user_id==User.user_id', cascade='all')
     repo_to_perm = relationship('UserRepoToPerm', primaryjoin='UserRepoToPerm.user_id==User.user_id', cascade='all')
 
     group_member = relationship('UsersGroupMember', cascade='all')
 
     @property
     def full_contact(self):
         return '%s %s <%s>' % (self.name, self.lastname, self.email)
 
     @property
     def short_contact(self):
         return '%s %s' % (self.name, self.lastname)
 
     @property
     def is_admin(self):
         return self.admin
 
     def __repr__(self):
         try:
             return "<%s('id:%s:%s')>" % (self.__class__.__name__,
                                              self.user_id, self.username)
         except:
             return self.__class__.__name__
 
     @classmethod
     def get_by_username(cls, username, case_insensitive=False):
         if case_insensitive:
             return Session.query(cls).filter(cls.username.ilike(username)).scalar()
         else:
             return Session.query(cls).filter(cls.username == username).scalar()
 
     @classmethod
     def get_by_api_key(cls, api_key):
         return cls.query().filter(cls.api_key == api_key).one()
 
     def update_lastlogin(self):
         """Update user lastlogin"""
 
         self.last_login = datetime.datetime.now()
         Session.add(self)
         Session.commit()
         log.debug('updated user %s lastlogin' % self.username)
 
     @classmethod
     def create(cls, form_data):
         from rhodecode.lib.auth import get_crypt_password
 
         try:
             new_user = cls()
             for k, v in form_data.items():
                 if k == 'password':
                     v = get_crypt_password(v)
                 setattr(new_user, k, v)
 
             new_user.api_key = generate_api_key(form_data['username'])
             Session.add(new_user)
             Session.commit()
             return new_user
         except:
             log.error(traceback.format_exc())
             Session.rollback()
             raise
 
 class UserLog(Base, BaseModel):
     __tablename__ = 'user_logs'
     __table_args__ = {'extend_existing':True}
     user_log_id = Column("user_log_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
     repository_name = Column("repository_name", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     user_ip = Column("user_ip", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     action = Column("action", UnicodeText(length=1200000, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     action_date = Column("action_date", DateTime(timezone=False), nullable=True, unique=None, default=None)
 
     @property
     def action_as_day(self):
         return date(*self.action_date.timetuple()[:3])
 
     user = relationship('User')
     repository = relationship('Repository')
 
 
 class UsersGroup(Base, BaseModel):
     __tablename__ = 'users_groups'
     __table_args__ = {'extend_existing':True}
 
     users_group_id = Column("users_group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_name = Column("users_group_name", String(length=255, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
     users_group_active = Column("users_group_active", Boolean(), nullable=True, unique=None, default=None)
 
     members = relationship('UsersGroupMember', cascade="all, delete, delete-orphan", lazy="joined")
 
     def __repr__(self):
         return '<userGroup(%s)>' % (self.users_group_name)
 
     @classmethod
     def get_by_group_name(cls, group_name, cache=False, case_insensitive=False):
         if case_insensitive:
             gr = cls.query()\
                 .filter(cls.users_group_name.ilike(group_name))
         else:
             gr = cls.query()\
                 .filter(cls.users_group_name == group_name)
         if cache:
             gr = gr.options(FromCache("sql_cache_short",
                                           "get_user_%s" % group_name))
         return gr.scalar()
 
 
     @classmethod
     def get(cls, users_group_id, cache=False):
         users_group = cls.query()
         if cache:
             users_group = users_group.options(FromCache("sql_cache_short",
                                     "get_users_group_%s" % users_group_id))
         return users_group.get(users_group_id)
 
     @classmethod
     def create(cls, form_data):
         try:
             new_users_group = cls()
             for k, v in form_data.items():
                 setattr(new_users_group, k, v)
 
             Session.add(new_users_group)
             Session.commit()
             return new_users_group
         except:
             log.error(traceback.format_exc())
             Session.rollback()
             raise
 
     @classmethod
     def update(cls, users_group_id, form_data):
 
         try:
             users_group = cls.get(users_group_id, cache=False)
 
             for k, v in form_data.items():
                 if k == 'users_group_members':
                     users_group.members = []
                     Session.flush()
                     members_list = []
                     if v:
                         v = [v] if isinstance(v, basestring) else v
                         for u_id in set(v):
                             member = UsersGroupMember(users_group_id, u_id)
                             members_list.append(member)
                     setattr(users_group, 'members', members_list)
                 setattr(users_group, k, v)
 
             Session.add(users_group)
             Session.commit()
         except:
             log.error(traceback.format_exc())
             Session.rollback()
             raise
 
     @classmethod
     def delete(cls, users_group_id):
         try:
 
             # check if this group is not assigned to repo
             assigned_groups = UsersGroupRepoToPerm.query()\
                 .filter(UsersGroupRepoToPerm.users_group_id ==
                         users_group_id).all()
 
             if assigned_groups:
                 raise UsersGroupsAssignedException('RepoGroup assigned to %s' %
                                                    assigned_groups)
 
             users_group = cls.get(users_group_id, cache=False)
             Session.delete(users_group)
             Session.commit()
         except:
             log.error(traceback.format_exc())
             Session.rollback()
             raise
 
 class UsersGroupMember(Base, BaseModel):
     __tablename__ = 'users_groups_members'
     __table_args__ = {'extend_existing':True}
 
     users_group_member_id = Column("users_group_member_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User', lazy='joined')
     users_group = relationship('UsersGroup')
 
     def __init__(self, gr_id='', u_id=''):
         self.users_group_id = gr_id
         self.user_id = u_id
 
     @staticmethod
     def add_user_to_group(group, user):
         ugm = UsersGroupMember()
         ugm.users_group = group
         ugm.user = user
         Session.add(ugm)
         Session.commit()
         return ugm
 
 class Repository(Base, BaseModel):
     __tablename__ = 'repositories'
     __table_args__ = (UniqueConstraint('repo_name'), {'extend_existing':True},)
 
     repo_id = Column("repo_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     repo_name = Column("repo_name", String(length=255, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
     clone_uri = Column("clone_uri", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=False, default=None)
     repo_type = Column("repo_type", String(length=255, convert_unicode=False, assert_unicode=None), nullable=False, unique=False, default='hg')
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=False, default=None)
     private = Column("private", Boolean(), nullable=True, unique=None, default=None)
     enable_statistics = Column("statistics", Boolean(), nullable=True, unique=None, default=True)
     enable_downloads = Column("downloads", Boolean(), nullable=True, unique=None, default=True)
     description = Column("description", String(length=10000, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     created_on = Column('created_on', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)
 
     fork_id = Column("fork_id", Integer(), ForeignKey('repositories.repo_id'), nullable=True, unique=False, default=None)
     group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=True, unique=False, default=None)
 
 
     user = relationship('User')
     fork = relationship('Repository', remote_side=repo_id)
     group = relationship('RepoGroup')
     repo_to_perm = relationship('UserRepoToPerm', cascade='all', order_by='UserRepoToPerm.repo_to_perm_id')
     users_group_to_perm = relationship('UsersGroupRepoToPerm', cascade='all')
     stats = relationship('Statistics', cascade='all', uselist=False)
 
     followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_repo_id==Repository.repo_id', cascade='all')
 
     logs = relationship('UserLog', cascade='all')
 
     def __repr__(self):
         return "<%s('%s:%s')>" % (self.__class__.__name__,
                                   self.repo_id, self.repo_name)
 
     @classmethod
     def url_sep(cls):
         return '/'
 
     @classmethod
     def get_by_repo_name(cls, repo_name):
         q = Session.query(cls).filter(cls.repo_name == repo_name)
         q = q.options(joinedload(Repository.fork))\
                 .options(joinedload(Repository.user))\
                 .options(joinedload(Repository.group))
         return q.one()
 
     @classmethod
     def get_repo_forks(cls, repo_id):
         return cls.query().filter(Repository.fork_id == repo_id)
 
     @classmethod
     def base_path(cls):
         """
         Returns base path when all repos are stored
 
         :param cls:
         """
         q = Session.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key ==
                                               cls.url_sep())
         q.options(FromCache("sql_cache_short", "repository_repo_path"))
         return q.one().ui_value
 
     @property
     def just_name(self):
         return self.repo_name.split(Repository.url_sep())[-1]
 
     @property
     def groups_with_parents(self):
         groups = []
         if self.group is None:
             return groups
 
         cur_gr = self.group
         groups.insert(0, cur_gr)
         while 1:
             gr = getattr(cur_gr, 'parent_group', None)
             cur_gr = cur_gr.parent_group
             if gr is None:
                 break
             groups.insert(0, gr)
 
         return groups
 
     @property
     def groups_and_repo(self):
         return self.groups_with_parents, self.just_name
 
     @LazyProperty
     def repo_path(self):
         """
         Returns base full path for that repository means where it actually
         exists on a filesystem
         """
         q = Session.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key ==
                                               Repository.url_sep())
         q.options(FromCache("sql_cache_short", "repository_repo_path"))
         return q.one().ui_value
 
     @property
     def repo_full_path(self):
         p = [self.repo_path]
         # we need to split the name by / since this is how we store the
         # names in the database, but that eventually needs to be converted
         # into a valid system path
         p += self.repo_name.split(Repository.url_sep())
         return os.path.join(*p)
 
     def get_new_name(self, repo_name):
         """
         returns new full repository name based on assigned group and new new
 
         :param group_name:
         """
         path_prefix = self.group.full_path_splitted if self.group else []
         return Repository.url_sep().join(path_prefix + [repo_name])
 
     @property
     def _ui(self):
         """
         Creates an db based ui object for this repository
         """
         from mercurial import ui
         from mercurial import config
         baseui = ui.ui()
 
         #clean the baseui object
         baseui._ocfg = config.config()
         baseui._ucfg = config.config()
         baseui._tcfg = config.config()
 
 
         ret = RhodeCodeUi.query()\
             .options(FromCache("sql_cache_short", "repository_repo_ui")).all()
 
         hg_ui = ret
         for ui_ in hg_ui:
             if ui_.ui_active:
                 log.debug('settings ui from db[%s]%s:%s', ui_.ui_section,
                           ui_.ui_key, ui_.ui_value)
                 baseui.setconfig(ui_.ui_section, ui_.ui_key, ui_.ui_value)
 
         return baseui
 
     @classmethod
     def is_valid(cls, repo_name):
         """
         returns True if given repo name is a valid filesystem repository
 
         :param cls:
         :param repo_name:
         """
         from rhodecode.lib.utils import is_valid_repo
 
         return is_valid_repo(repo_name, cls.base_path())
 
 
     #==========================================================================
     # SCM PROPERTIES
     #==========================================================================
 
     def get_changeset(self, rev):
         return get_changeset_safe(self.scm_instance, rev)
 
     @property
     def tip(self):
         return self.get_changeset('tip')
 
     @property
     def author(self):
         return self.tip.author
 
     @property
     def last_change(self):
         return self.scm_instance.last_change
 
     #==========================================================================
     # SCM CACHE INSTANCE
     #==========================================================================
 
     @property
     def invalidate(self):
         return CacheInvalidation.invalidate(self.repo_name)
 
     def set_invalidate(self):
         """
         set a cache for invalidation for this instance
         """
         CacheInvalidation.set_invalidate(self.repo_name)
 
     @LazyProperty
     def scm_instance(self):
         return self.__get_instance()
 
     @property
     def scm_instance_cached(self):
         @cache_region('long_term')
         def _c(repo_name):
             return self.__get_instance()
         rn = self.repo_name
 
         inv = self.invalidate
         if inv is not None:
             region_invalidate(_c, None, rn)
             # update our cache
             CacheInvalidation.set_valid(inv.cache_key)
         return _c(rn)
 
     def __get_instance(self):
 
         repo_full_path = self.repo_full_path
 
         try:
             alias = get_scm(repo_full_path)[0]
             log.debug('Creating instance of %s repository' % alias)
             backend = get_backend(alias)
         except VCSError:
             log.error(traceback.format_exc())
             log.error('Perhaps this repository is in db and not in '
                       'filesystem run rescan repositories with '
                       '"destroy old data " option from admin panel')
             return
 
         if alias == 'hg':
 
             repo = backend(safe_str(repo_full_path), create=False,
                            baseui=self._ui)
             # skip hidden web repository
             if repo._get_hidden():
                 return
         else:
             repo = backend(repo_full_path, create=False)
 
         return repo
 
 
 class RepoGroup(Base, BaseModel):
     __tablename__ = 'groups'
     __table_args__ = (UniqueConstraint('group_name', 'group_parent_id'),
                       CheckConstraint('group_id != group_parent_id'), {'extend_existing':True},)
     __mapper_args__ = {'order_by':'group_name'}
 
     group_id = Column("group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     group_name = Column("group_name", String(length=255, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
     group_parent_id = Column("group_parent_id", Integer(), ForeignKey('groups.group_id'), nullable=True, unique=None, default=None)
     group_description = Column("group_description", String(length=10000, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
     parent_group = relationship('RepoGroup', remote_side=group_id)
 
 
     def __init__(self, group_name='', parent_group=None):
         self.group_name = group_name
         self.parent_group = parent_group
 
     def __repr__(self):
         return "<%s('%s:%s')>" % (self.__class__.__name__, self.group_id,
                                   self.group_name)
 
     @classmethod
     def groups_choices(cls):
         from webhelpers.html import literal as _literal
         repo_groups = [('', '')]
         sep = ' &raquo; '
         _name = lambda k: _literal(sep.join(k))
 
         repo_groups.extend([(x.group_id, _name(x.full_path_splitted))
                               for x in cls.query().all()])
 
         repo_groups = sorted(repo_groups, key=lambda t: t[1].split(sep)[0])
         return repo_groups
 
     @classmethod
     def url_sep(cls):
         return '/'
 
     @classmethod
     def get_by_group_name(cls, group_name, cache=False, case_insensitive=False):
         if case_insensitive:
             gr = cls.query()\
                 .filter(cls.group_name.ilike(group_name))
         else:
             gr = cls.query()\
                 .filter(cls.group_name == group_name)
         if cache:
             gr = gr.options(FromCache("sql_cache_short",
                                           "get_group_%s" % group_name))
         return gr.scalar()
 
     @property
     def parents(self):
         parents_recursion_limit = 5
         groups = []
         if self.parent_group is None:
             return groups
         cur_gr = self.parent_group
         groups.insert(0, cur_gr)
         cnt = 0
         while 1:
             cnt += 1
             gr = getattr(cur_gr, 'parent_group', None)
             cur_gr = cur_gr.parent_group
             if gr is None:
                 break
             if cnt == parents_recursion_limit:
                 # this will prevent accidental infinit loops
                 log.error('group nested more than %s' %
                           parents_recursion_limit)
                 break
 
             groups.insert(0, gr)
         return groups
 
     @property
     def children(self):
         return Group.query().filter(Group.parent_group == self)
 
     @property
     def name(self):
         return self.group_name.split(Group.url_sep())[-1]
 
     @property
     def full_path(self):
         return self.group_name
 
     @property
     def full_path_splitted(self):
         return self.group_name.split(Group.url_sep())
 
     @property
     def repositories(self):
         return Repository.query().filter(Repository.group == self)
 
     @property
     def repositories_recursive_count(self):
         cnt = self.repositories.count()
 
         def children_count(group):
             cnt = 0
             for child in group.children:
                 cnt += child.repositories.count()
                 cnt += children_count(child)
             return cnt
 
         return cnt + children_count(self)
 
 
     def get_new_name(self, group_name):
         """
         returns new full group name based on parent and new name
 
         :param group_name:
         """
         path_prefix = (self.parent_group.full_path_splitted if
                        self.parent_group else [])
         return Group.url_sep().join(path_prefix + [group_name])
 
 
 class Permission(Base, BaseModel):
     __tablename__ = 'permissions'
     __table_args__ = {'extend_existing':True}
     permission_id = Column("permission_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     permission_name = Column("permission_name", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     permission_longname = Column("permission_longname", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
     def __repr__(self):
         return "<%s('%s:%s')>" % (self.__class__.__name__,
                                   self.permission_id, self.permission_name)
 
     @classmethod
     def get_by_key(cls, key):
         return cls.query().filter(cls.permission_name == key).scalar()
 
 class UserRepoToPerm(Base, BaseModel):
     __tablename__ = 'repo_to_perm'
     __table_args__ = (UniqueConstraint('user_id', 'repository_id'), {'extend_existing':True})
     repo_to_perm_id = Column("repo_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User')
     permission = relationship('Permission')
     repository = relationship('Repository')
 
 class UserToPerm(Base, BaseModel):
     __tablename__ = 'user_to_perm'
     __table_args__ = (UniqueConstraint('user_id', 'permission_id'), {'extend_existing':True})
     user_to_perm_id = Column("user_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User')
     permission = relationship('Permission')
 
     @classmethod
     def has_perm(cls, user_id, perm):
         if not isinstance(perm, Permission):
             raise Exception('perm needs to be an instance of Permission class')
 
         return cls.query().filter(cls.user_id == user_id)\
             .filter(cls.permission == perm).scalar() is not None
 
     @classmethod
     def grant_perm(cls, user_id, perm):
         if not isinstance(perm, Permission):
             raise Exception('perm needs to be an instance of Permission class')
 
         new = cls()
         new.user_id = user_id
         new.permission = perm
         try:
             Session.add(new)
             Session.commit()
         except:
             Session.rollback()
 
 
     @classmethod
     def revoke_perm(cls, user_id, perm):
         if not isinstance(perm, Permission):
             raise Exception('perm needs to be an instance of Permission class')
 
         try:
             cls.query().filter(cls.user_id == user_id)\
                 .filter(cls.permission == perm).delete()
             Session.commit()
         except:
             Session.rollback()
 
 class UsersGroupRepoToPerm(Base, BaseModel):
     __tablename__ = 'users_group_repo_to_perm'
     __table_args__ = (UniqueConstraint('repository_id', 'users_group_id', 'permission_id'), {'extend_existing':True})
     users_group_to_perm_id = Column("users_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
 
     users_group = relationship('UsersGroup')
     permission = relationship('Permission')
     repository = relationship('Repository')
 
     def __repr__(self):
         return '<userGroup:%s => %s >' % (self.users_group, self.repository)
 
 class UsersGroupToPerm(Base, BaseModel):
     __tablename__ = 'users_group_to_perm'
     __table_args__ = {'extend_existing':True}
     users_group_to_perm_id = Column("users_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
 
     users_group = relationship('UsersGroup')
     permission = relationship('Permission')
 
 
     @classmethod
     def has_perm(cls, users_group_id, perm):
         if not isinstance(perm, Permission):
             raise Exception('perm needs to be an instance of Permission class')
 
         return cls.query().filter(cls.users_group_id ==
                                          users_group_id)\
                                          .filter(cls.permission == perm)\
                                          .scalar() is not None
 
     @classmethod
     def grant_perm(cls, users_group_id, perm):
         if not isinstance(perm, Permission):
             raise Exception('perm needs to be an instance of Permission class')
 
         new = cls()
         new.users_group_id = users_group_id
         new.permission = perm
         try:
             Session.add(new)
             Session.commit()
         except:
             Session.rollback()
 
 
     @classmethod
     def revoke_perm(cls, users_group_id, perm):
         if not isinstance(perm, Permission):
             raise Exception('perm needs to be an instance of Permission class')
 
         try:
             cls.query().filter(cls.users_group_id == users_group_id)\
                 .filter(cls.permission == perm).delete()
             Session.commit()
         except:
             Session.rollback()
 
 
 class UserRepoGroupToPerm(Base, BaseModel):
     __tablename__ = 'group_to_perm'
     __table_args__ = (UniqueConstraint('group_id', 'permission_id'), {'extend_existing':True})
 
     group_to_perm_id = Column("group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
     group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User')
     permission = relationship('Permission')
     group = relationship('RepoGroup')
 
 class Statistics(Base, BaseModel):
     __tablename__ = 'statistics'
     __table_args__ = (UniqueConstraint('repository_id'), {'extend_existing':True})
     stat_id = Column("stat_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=True, default=None)
     stat_on_revision = Column("stat_on_revision", Integer(), nullable=False)
     commit_activity = Column("commit_activity", LargeBinary(1000000), nullable=False)#JSON data
     commit_activity_combined = Column("commit_activity_combined", LargeBinary(), nullable=False)#JSON data
     languages = Column("languages", LargeBinary(1000000), nullable=False)#JSON data
 
     repository = relationship('Repository', single_parent=True)
 
 class UserFollowing(Base, BaseModel):
     __tablename__ = 'user_followings'
     __table_args__ = (UniqueConstraint('user_id', 'follows_repository_id'),
                       UniqueConstraint('user_id', 'follows_user_id')
                       , {'extend_existing':True})
 
     user_following_id = Column("user_following_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     follows_repo_id = Column("follows_repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=True, unique=None, default=None)
     follows_user_id = Column("follows_user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
     follows_from = Column('follows_from', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)
 
     user = relationship('User', primaryjoin='User.user_id==UserFollowing.user_id')
 
     follows_user = relationship('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')
     follows_repository = relationship('Repository', order_by='Repository.repo_name')
 
 
     @classmethod
     def get_repo_followers(cls, repo_id):
         return cls.query().filter(cls.follows_repo_id == repo_id)
 
 class CacheInvalidation(Base, BaseModel):
     __tablename__ = 'cache_invalidation'
     __table_args__ = (UniqueConstraint('cache_key'), {'extend_existing':True})
     cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     cache_key = Column("cache_key", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     cache_args = Column("cache_args", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False)
 
 
     def __init__(self, cache_key, cache_args=''):
         self.cache_key = cache_key
         self.cache_args = cache_args
         self.cache_active = False
 
     def __repr__(self):
         return "<%s('%s:%s')>" % (self.__class__.__name__,
                                   self.cache_id, self.cache_key)
 
     @classmethod
     def invalidate(cls, key):
         """
         Returns Invalidation object if this given key should be invalidated
         None otherwise. `cache_active = False` means that this cache
         state is not valid and needs to be invalidated
 
         :param key:
         """
         return cls.query()\
                 .filter(CacheInvalidation.cache_key == key)\
                 .filter(CacheInvalidation.cache_active == False)\
                 .scalar()
 
     @classmethod
     def set_invalidate(cls, key):
         """
         Mark this Cache key for invalidation
 
         :param key:
         """
 
         log.debug('marking %s for invalidation' % key)
         inv_obj = Session.query(cls)\
             .filter(cls.cache_key == key).scalar()
         if inv_obj:
             inv_obj.cache_active = False
         else:
             log.debug('cache key not found in invalidation db -> creating one')
             inv_obj = CacheInvalidation(key)
 
         try:
             Session.add(inv_obj)
             Session.commit()
         except Exception:
             log.error(traceback.format_exc())
             Session.rollback()
 
     @classmethod
     def set_valid(cls, key):
         """
         Mark this cache key as active and currently cached
 
         :param key:
         """
         inv_obj = Session.query(CacheInvalidation)\
             .filter(CacheInvalidation.cache_key == key).scalar()
         inv_obj.cache_active = True
         Session.add(inv_obj)
         Session.commit()
 
 class DbMigrateVersion(Base, BaseModel):
     __tablename__ = 'db_migrate_version'
     __table_args__ = {'extend_existing':True}
     repository_id = Column('repository_id', String(250), primary_key=True)
     repository_path = Column('repository_path', Text)
     version = Column('version', Integer)

rhodecode/lib/diffs.py

@@ -1,517 +1,517 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.diffs
     ~~~~~~~~~~~~~~~~~~~
 
     Set of diffing helpers, previously part of vcs
 
 
     :created_on: Dec 4, 2011
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :original copyright: 2007-2008 by Armin Ronacher
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import re
 import difflib
 import markupsafe
 from itertools import tee, imap
 
 from pylons.i18n.translation import _
 
-from vcs.exceptions import VCSError
-from vcs.nodes import FileNode
+from rhodecode.lib.vcs.exceptions import VCSError
+from rhodecode.lib.vcs.nodes import FileNode
 
 from rhodecode.lib.utils import EmptyChangeset
 
 
 def wrap_to_table(str_):
     return '''<table class="code-difftable">
                 <tr class="line no-comment">
                 <td class="lineno new"></td>
                 <td class="code no-comment"><pre>%s</pre></td>
                 </tr>
               </table>''' % str_
 
 
 def wrapped_diff(filenode_old, filenode_new, cut_off_limit=None,
                 ignore_whitespace=True, line_context=3,
                 enable_comments=False):
     """
     returns a wrapped diff into a table, checks for cut_off_limit and presents
     proper message
     """
 
     if filenode_old is None:
         filenode_old = FileNode(filenode_new.path, '', EmptyChangeset())
 
     if filenode_old.is_binary or filenode_new.is_binary:
         diff = wrap_to_table(_('binary file'))
         stats = (0, 0)
         size = 0
 
     elif cut_off_limit != -1 and (cut_off_limit is None or
     (filenode_old.size < cut_off_limit and filenode_new.size < cut_off_limit)):
 
         f_gitdiff = get_gitdiff(filenode_old, filenode_new,
                                 ignore_whitespace=ignore_whitespace,
                                 context=line_context)
         diff_processor = DiffProcessor(f_gitdiff, format='gitdiff')
 
         diff = diff_processor.as_html(enable_comments=enable_comments)
         stats = diff_processor.stat()
         size = len(diff or '')
     else:
         diff = wrap_to_table(_('Changeset was to big and was cut off, use '
                                'diff menu to display this diff'))
         stats = (0, 0)
         size = 0
 
     if not diff:
         diff = wrap_to_table(_('No changes detected'))
 
     cs1 = filenode_old.last_changeset.raw_id
     cs2 = filenode_new.last_changeset.raw_id
 
     return size, cs1, cs2, diff, stats
 
 
 def get_gitdiff(filenode_old, filenode_new, ignore_whitespace=True, context=3):
     """
     Returns git style diff between given ``filenode_old`` and ``filenode_new``.
 
     :param ignore_whitespace: ignore whitespaces in diff
     """
     # make sure we pass in default context
     context = context or 3
 
     for filenode in (filenode_old, filenode_new):
         if not isinstance(filenode, FileNode):
             raise VCSError("Given object should be FileNode object, not %s"
                 % filenode.__class__)
 
     repo = filenode_new.changeset.repository
     old_raw_id = getattr(filenode_old.changeset, 'raw_id', repo.EMPTY_CHANGESET)
     new_raw_id = getattr(filenode_new.changeset, 'raw_id', repo.EMPTY_CHANGESET)
 
     vcs_gitdiff = repo.get_diff(old_raw_id, new_raw_id, filenode_new.path,
                                  ignore_whitespace, context)
 
     return vcs_gitdiff
 
 
 class DiffProcessor(object):
     """
     Give it a unified diff and it returns a list of the files that were
     mentioned in the diff together with a dict of meta information that
     can be used to render it in a HTML template.
     """
     _chunk_re = re.compile(r'@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)')
 
     def __init__(self, diff, differ='diff', format='udiff'):
         """
         :param diff:   a text in diff format or generator
         :param format: format of diff passed, `udiff` or `gitdiff`
         """
         if isinstance(diff, basestring):
             diff = [diff]
 
         self.__udiff = diff
         self.__format = format
         self.adds = 0
         self.removes = 0
 
         if isinstance(self.__udiff, basestring):
             self.lines = iter(self.__udiff.splitlines(1))
 
         elif self.__format == 'gitdiff':
             udiff_copy = self.copy_iterator()
             self.lines = imap(self.escaper, self._parse_gitdiff(udiff_copy))
         else:
             udiff_copy = self.copy_iterator()
             self.lines = imap(self.escaper, udiff_copy)
 
         # Select a differ.
         if differ == 'difflib':
             self.differ = self._highlight_line_difflib
         else:
             self.differ = self._highlight_line_udiff
 
     def escaper(self, string):
         return markupsafe.escape(string)
 
     def copy_iterator(self):
         """
         make a fresh copy of generator, we should not iterate thru
         an original as it's needed for repeating operations on
         this instance of DiffProcessor
         """
         self.__udiff, iterator_copy = tee(self.__udiff)
         return iterator_copy
 
     def _extract_rev(self, line1, line2):
         """
         Extract the filename and revision hint from a line.
         """
 
         try:
             if line1.startswith('--- ') and line2.startswith('+++ '):
                 l1 = line1[4:].split(None, 1)
                 old_filename = (l1[0].replace('a/', '', 1)
                                 if len(l1) >= 1 else None)
                 old_rev = l1[1] if len(l1) == 2 else 'old'
 
                 l2 = line2[4:].split(None, 1)
                 new_filename = (l2[0].replace('b/', '', 1)
                                 if len(l1) >= 1 else None)
                 new_rev = l2[1] if len(l2) == 2 else 'new'
 
                 filename = (old_filename
                             if old_filename != '/dev/null' else new_filename)
 
                 return filename, new_rev, old_rev
         except (ValueError, IndexError):
             pass
 
         return None, None, None
 
     def _parse_gitdiff(self, diffiterator):
         def line_decoder(l):
             if l.startswith('+') and not l.startswith('+++'):
                 self.adds += 1
             elif l.startswith('-') and not l.startswith('---'):
                 self.removes += 1
             return l.decode('utf8', 'replace')
 
         output = list(diffiterator)
         size = len(output)
 
         if size == 2:
             l = []
             l.extend([output[0]])
             l.extend(output[1].splitlines(1))
             return map(line_decoder, l)
         elif size == 1:
             return  map(line_decoder, output[0].splitlines(1))
         elif size == 0:
             return []
 
         raise Exception('wrong size of diff %s' % size)
 
     def _highlight_line_difflib(self, line, next_):
         """
         Highlight inline changes in both lines.
         """
 
         if line['action'] == 'del':
             old, new = line, next_
         else:
             old, new = next_, line
 
         oldwords = re.split(r'(\W)', old['line'])
         newwords = re.split(r'(\W)', new['line'])
 
         sequence = difflib.SequenceMatcher(None, oldwords, newwords)
 
         oldfragments, newfragments = [], []
         for tag, i1, i2, j1, j2 in sequence.get_opcodes():
             oldfrag = ''.join(oldwords[i1:i2])
             newfrag = ''.join(newwords[j1:j2])
             if tag != 'equal':
                 if oldfrag:
                     oldfrag = '<del>%s</del>' % oldfrag
                 if newfrag:
                     newfrag = '<ins>%s</ins>' % newfrag
             oldfragments.append(oldfrag)
             newfragments.append(newfrag)
 
         old['line'] = "".join(oldfragments)
         new['line'] = "".join(newfragments)
 
     def _highlight_line_udiff(self, line, next_):
         """
         Highlight inline changes in both lines.
         """
         start = 0
         limit = min(len(line['line']), len(next_['line']))
         while start < limit and line['line'][start] == next_['line'][start]:
             start += 1
         end = -1
         limit -= start
         while -end <= limit and line['line'][end] == next_['line'][end]:
             end -= 1
         end += 1
         if start or end:
             def do(l):
                 last = end + len(l['line'])
                 if l['action'] == 'add':
                     tag = 'ins'
                 else:
                     tag = 'del'
                 l['line'] = '%s<%s>%s</%s>%s' % (
                     l['line'][:start],
                     tag,
                     l['line'][start:last],
                     tag,
                     l['line'][last:]
                 )
             do(line)
             do(next_)
 
     def _parse_udiff(self):
         """
         Parse the diff an return data for the template.
         """
         lineiter = self.lines
         files = []
         try:
             line = lineiter.next()
             # skip first context
             skipfirst = True
             while 1:
                 # continue until we found the old file
                 if not line.startswith('--- '):
                     line = lineiter.next()
                     continue
 
                 chunks = []
                 filename, old_rev, new_rev = \
                     self._extract_rev(line, lineiter.next())
                 files.append({
                     'filename':         filename,
                     'old_revision':     old_rev,
                     'new_revision':     new_rev,
                     'chunks':           chunks
                 })
 
                 line = lineiter.next()
                 while line:
                     match = self._chunk_re.match(line)
                     if not match:
                         break
 
                     lines = []
                     chunks.append(lines)
 
                     old_line, old_end, new_line, new_end = \
                         [int(x or 1) for x in match.groups()[:-1]]
                     old_line -= 1
                     new_line -= 1
                     context = len(match.groups()) == 5
                     old_end += old_line
                     new_end += new_line
 
                     if context:
                         if not skipfirst:
                             lines.append({
                                 'old_lineno': '...',
                                 'new_lineno': '...',
                                 'action':     'context',
                                 'line':       line,
                             })
                         else:
                             skipfirst = False
 
                     line = lineiter.next()
                     while old_line < old_end or new_line < new_end:
                         if line:
                             command, line = line[0], line[1:]
                         else:
                             command = ' '
                         affects_old = affects_new = False
 
                         # ignore those if we don't expect them
                         if command in '#@':
                             continue
                         elif command == '+':
                             affects_new = True
                             action = 'add'
                         elif command == '-':
                             affects_old = True
                             action = 'del'
                         else:
                             affects_old = affects_new = True
                             action = 'unmod'
 
                         old_line += affects_old
                         new_line += affects_new
                         lines.append({
                             'old_lineno':   affects_old and old_line or '',
                             'new_lineno':   affects_new and new_line or '',
                             'action':       action,
                             'line':         line
                         })
                         line = lineiter.next()
 
         except StopIteration:
             pass
 
         # highlight inline changes
         for _ in files:
             for chunk in chunks:
                 lineiter = iter(chunk)
                 #first = True
                 try:
                     while 1:
                         line = lineiter.next()
                         if line['action'] != 'unmod':
                             nextline = lineiter.next()
                             if nextline['action'] == 'unmod' or \
                                nextline['action'] == line['action']:
                                 continue
                             self.differ(line, nextline)
                 except StopIteration:
                     pass
 
         return files
 
     def prepare(self):
         """
         Prepare the passed udiff for HTML rendering. It'l return a list
         of dicts
         """
         return self._parse_udiff()
 
     def _safe_id(self, idstring):
         """Make a string safe for including in an id attribute.
 
         The HTML spec says that id attributes 'must begin with
         a letter ([A-Za-z]) and may be followed by any number
         of letters, digits ([0-9]), hyphens ("-"), underscores
         ("_"), colons (":"), and periods (".")'. These regexps
         are slightly over-zealous, in that they remove colons
         and periods unnecessarily.
 
         Whitespace is transformed into underscores, and then
         anything which is not a hyphen or a character that
         matches \w (alphanumerics and underscore) is removed.
 
         """
         # Transform all whitespace to underscore
         idstring = re.sub(r'\s', "_", '%s' % idstring)
         # Remove everything that is not a hyphen or a member of \w
         idstring = re.sub(r'(?!-)\W', "", idstring).lower()
         return idstring
 
     def raw_diff(self):
         """
         Returns raw string as udiff
         """
         udiff_copy = self.copy_iterator()
         if self.__format == 'gitdiff':
             udiff_copy = self._parse_gitdiff(udiff_copy)
         return u''.join(udiff_copy)
 
     def as_html(self, table_class='code-difftable', line_class='line',
                 new_lineno_class='lineno old', old_lineno_class='lineno new',
                 code_class='code', enable_comments=False):
         """
         Return udiff as html table with customized css classes
         """
         def _link_to_if(condition, label, url):
             """
             Generates a link if condition is meet or just the label if not.
             """
 
             if condition:
                 return '''<a href="%(url)s">%(label)s</a>''' % {
                     'url': url,
                     'label': label
                 }
             else:
                 return label
         diff_lines = self.prepare()
         _html_empty = True
         _html = []
         _html.append('''<table class="%(table_class)s">\n''' % {
             'table_class': table_class
         })
         for diff in diff_lines:
             for line in diff['chunks']:
                 _html_empty = False
                 for change in line:
                     _html.append('''<tr class="%(lc)s %(action)s">\n''' % {
                         'lc': line_class,
                         'action': change['action']
                     })
                     anchor_old_id = ''
                     anchor_new_id = ''
                     anchor_old = "%(filename)s_o%(oldline_no)s" % {
                         'filename': self._safe_id(diff['filename']),
                         'oldline_no': change['old_lineno']
                     }
                     anchor_new = "%(filename)s_n%(oldline_no)s" % {
                         'filename': self._safe_id(diff['filename']),
                         'oldline_no': change['new_lineno']
                     }
                     cond_old = (change['old_lineno'] != '...' and
                                 change['old_lineno'])
                     cond_new = (change['new_lineno'] != '...' and
                                 change['new_lineno'])
                     if cond_old:
                         anchor_old_id = 'id="%s"' % anchor_old
                     if cond_new:
                         anchor_new_id = 'id="%s"' % anchor_new
                     ###########################################################
                     # OLD LINE NUMBER
                     ###########################################################
                     _html.append('''\t<td %(a_id)s class="%(olc)s">''' % {
                         'a_id': anchor_old_id,
                         'olc': old_lineno_class
                     })
 
                     _html.append('''%(link)s''' % {
                         'link': _link_to_if(True, change['old_lineno'],
                                             '#%s' % anchor_old)
                     })
                     _html.append('''</td>\n''')
                     ###########################################################
                     # NEW LINE NUMBER
                     ###########################################################
 
                     _html.append('''\t<td %(a_id)s class="%(nlc)s">''' % {
                         'a_id': anchor_new_id,
                         'nlc': new_lineno_class
                     })
 
                     _html.append('''%(link)s''' % {
                         'link': _link_to_if(True, change['new_lineno'],
                                             '#%s' % anchor_new)
                     })
                     _html.append('''</td>\n''')
                     ###########################################################
                     # CODE
                     ###########################################################
                     comments = '' if enable_comments else 'no-comment'
                     _html.append('''\t<td class="%(cc)s %(inc)s">''' % {
                         'cc': code_class,
                         'inc': comments
                     })
                     _html.append('''\n\t\t<pre>%(code)s</pre>\n''' % {
                         'code': change['line']
                     })
                     _html.append('''\t</td>''')
                     _html.append('''\n</tr>\n''')
         _html.append('''</table>''')
         if _html_empty:
             return None
         return ''.join(_html)
 
     def stat(self):
         """
         Returns tuple of added, and removed lines for this instance
         """
         return self.adds, self.removes

rhodecode/lib/helpers.py

@@ -1,886 +1,886 b''
 """Helper functions
 
 Consists of functions to typically be used within templates, but also
 available to Controllers. This module is available to both as 'h'.
 """
 import random
 import hashlib
 import StringIO
 import urllib
 import math
 import logging
 
 from datetime import datetime
 from pygments.formatters.html import HtmlFormatter
 from pygments import highlight as code_highlight
 from pylons import url, request, config
 from pylons.i18n.translation import _, ungettext
 from hashlib import md5
 
 from webhelpers.html import literal, HTML, escape
 from webhelpers.html.tools import *
 from webhelpers.html.builder import make_tag
 from webhelpers.html.tags import auto_discovery_link, checkbox, css_classes, \
     end_form, file, form, hidden, image, javascript_link, link_to, \
     link_to_if, link_to_unless, ol, required_legend, select, stylesheet_link, \
     submit, text, password, textarea, title, ul, xml_declaration, radio
 from webhelpers.html.tools import auto_link, button_to, highlight, \
     js_obfuscate, mail_to, strip_links, strip_tags, tag_re
 from webhelpers.number import format_byte_size, format_bit_size
 from webhelpers.pylonslib import Flash as _Flash
 from webhelpers.pylonslib.secure_form import secure_form
 from webhelpers.text import chop_at, collapse, convert_accented_entities, \
     convert_misc_entities, lchop, plural, rchop, remove_formatting, \
     replace_whitespace, urlify, truncate, wrap_paragraphs
 from webhelpers.date import time_ago_in_words
 from webhelpers.paginate import Page
 from webhelpers.html.tags import _set_input_attrs, _set_id_attr, \
     convert_boolean_attrs, NotGiven, _make_safe_id_component
 
 from rhodecode.lib.annotate import annotate_highlight
 from rhodecode.lib.utils import repo_name_slug
 from rhodecode.lib import str2bool, safe_unicode, safe_str, get_changeset_safe
 from rhodecode.lib.markup_renderer import MarkupRenderer
 
 log = logging.getLogger(__name__)
 
 
 def _reset(name, value=None, id=NotGiven, type="reset", **attrs):
     """
     Reset button
     """
     _set_input_attrs(attrs, type, name, value)
     _set_id_attr(attrs, id, name)
     convert_boolean_attrs(attrs, ["disabled"])
     return HTML.input(**attrs)
 
 reset = _reset
 safeid = _make_safe_id_component
 
 
 def FID(raw_id, path):
     """
     Creates a uniqe ID for filenode based on it's hash of path and revision
     it's safe to use in urls
 
     :param raw_id:
     :param path:
     """
 
     return 'C-%s-%s' % (short_id(raw_id), md5(path).hexdigest()[:12])
 
 
 def get_token():
     """Return the current authentication token, creating one if one doesn't
     already exist.
     """
     token_key = "_authentication_token"
     from pylons import session
     if not token_key in session:
         try:
             token = hashlib.sha1(str(random.getrandbits(128))).hexdigest()
         except AttributeError: # Python < 2.4
             token = hashlib.sha1(str(random.randrange(2 ** 128))).hexdigest()
         session[token_key] = token
         if hasattr(session, 'save'):
             session.save()
     return session[token_key]
 
 class _GetError(object):
     """Get error from form_errors, and represent it as span wrapped error
     message
 
     :param field_name: field to fetch errors for
     :param form_errors: form errors dict
     """
 
     def __call__(self, field_name, form_errors):
         tmpl = """<span class="error_msg">%s</span>"""
         if form_errors and form_errors.has_key(field_name):
             return literal(tmpl % form_errors.get(field_name))
 
 get_error = _GetError()
 
 class _ToolTip(object):
 
     def __call__(self, tooltip_title, trim_at=50):
         """Special function just to wrap our text into nice formatted
         autowrapped text
 
         :param tooltip_title:
         """
         return escape(tooltip_title)
 tooltip = _ToolTip()
 
 class _FilesBreadCrumbs(object):
 
     def __call__(self, repo_name, rev, paths):
         if isinstance(paths, str):
             paths = safe_unicode(paths)
         url_l = [link_to(repo_name, url('files_home',
                                         repo_name=repo_name,
                                         revision=rev, f_path=''))]
         paths_l = paths.split('/')
         for cnt, p in enumerate(paths_l):
             if p != '':
                 url_l.append(link_to(p,
                                      url('files_home',
                                          repo_name=repo_name,
                                          revision=rev,
                                          f_path='/'.join(paths_l[:cnt + 1])
                                          )
                                      )
                              )
 
         return literal('/'.join(url_l))
 
 files_breadcrumbs = _FilesBreadCrumbs()
 
 class CodeHtmlFormatter(HtmlFormatter):
     """My code Html Formatter for source codes
     """
 
     def wrap(self, source, outfile):
         return self._wrap_div(self._wrap_pre(self._wrap_code(source)))
 
     def _wrap_code(self, source):
         for cnt, it in enumerate(source):
             i, t = it
             t = '<div id="L%s">%s</div>' % (cnt + 1, t)
             yield i, t
 
     def _wrap_tablelinenos(self, inner):
         dummyoutfile = StringIO.StringIO()
         lncount = 0
         for t, line in inner:
             if t:
                 lncount += 1
             dummyoutfile.write(line)
 
         fl = self.linenostart
         mw = len(str(lncount + fl - 1))
         sp = self.linenospecial
         st = self.linenostep
         la = self.lineanchors
         aln = self.anchorlinenos
         nocls = self.noclasses
         if sp:
             lines = []
 
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if i % sp == 0:
                         if aln:
                             lines.append('<a href="#%s%d" class="special">%*d</a>' %
                                          (la, i, mw, i))
                         else:
                             lines.append('<span class="special">%*d</span>' % (mw, i))
                     else:
                         if aln:
                             lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                         else:
                             lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
         else:
             lines = []
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if aln:
                         lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                     else:
                         lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
 
         # in case you wonder about the seemingly redundant <div> here: since the
         # content in the other cell also is wrapped in a div, some browsers in
         # some configurations seem to mess up the formatting...
         if nocls:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td><div class="linenodiv" '
                       'style="background-color: #f0f0f0; padding-right: 10px">'
                       '<pre style="line-height: 125%">' +
                       ls + '</pre></div></td><td id="hlcode" class="code">')
         else:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td class="linenos"><div class="linenodiv"><pre>' +
                       ls + '</pre></div></td><td id="hlcode" class="code">')
         yield 0, dummyoutfile.getvalue()
         yield 0, '</td></tr></table>'
 
 
 def pygmentize(filenode, **kwargs):
     """pygmentize function using pygments
 
     :param filenode:
     """
 
     return literal(code_highlight(filenode.content,
                                   filenode.lexer, CodeHtmlFormatter(**kwargs)))
 
 
 def pygmentize_annotation(repo_name, filenode, **kwargs):
     """
     pygmentize function for annotation
 
     :param filenode:
     """
 
     color_dict = {}
 
     def gen_color(n=10000):
         """generator for getting n of evenly distributed colors using
         hsv color and golden ratio. It always return same order of colors
 
         :returns: RGB tuple
         """
 
         def hsv_to_rgb(h, s, v):
             if s == 0.0:
                 return v, v, v
             i = int(h * 6.0)  # XXX assume int() truncates!
             f = (h * 6.0) - i
             p = v * (1.0 - s)
             q = v * (1.0 - s * f)
             t = v * (1.0 - s * (1.0 - f))
             i = i % 6
             if i == 0:
                 return v, t, p
             if i == 1:
                 return q, v, p
             if i == 2:
                 return p, v, t
             if i == 3:
                 return p, q, v
             if i == 4:
                 return t, p, v
             if i == 5:
                 return v, p, q
 
         golden_ratio = 0.618033988749895
         h = 0.22717784590367374
 
         for _ in xrange(n):
             h += golden_ratio
             h %= 1
             HSV_tuple = [h, 0.95, 0.95]
             RGB_tuple = hsv_to_rgb(*HSV_tuple)
             yield map(lambda x: str(int(x * 256)), RGB_tuple)
 
     cgenerator = gen_color()
 
     def get_color_string(cs):
         if cs in color_dict:
             col = color_dict[cs]
         else:
             col = color_dict[cs] = cgenerator.next()
         return "color: rgb(%s)! important;" % (', '.join(col))
 
     def url_func(repo_name):
 
         def _url_func(changeset):
             author = changeset.author
             date = changeset.date
             message = tooltip(changeset.message)
 
             tooltip_html = ("<div style='font-size:0.8em'><b>Author:</b>"
                             " %s<br/><b>Date:</b> %s</b><br/><b>Message:"
                             "</b> %s<br/></div>")
 
             tooltip_html = tooltip_html % (author, date, message)
             lnk_format = '%5s:%s' % ('r%s' % changeset.revision,
                                      short_id(changeset.raw_id))
             uri = link_to(
                     lnk_format,
                     url('changeset_home', repo_name=repo_name,
                         revision=changeset.raw_id),
                     style=get_color_string(changeset.raw_id),
                     class_='tooltip',
                     title=tooltip_html
                   )
 
             uri += '\n'
             return uri
         return _url_func
 
     return literal(annotate_highlight(filenode, url_func(repo_name), **kwargs))
 
 
 def is_following_repo(repo_name, user_id):
     from rhodecode.model.scm import ScmModel
     return ScmModel().is_following_repo(repo_name, user_id)
 
 flash = _Flash()
 
 #==============================================================================
 # SCM FILTERS available via h.
 #==============================================================================
-from vcs.utils import author_name, author_email
+from rhodecode.lib.vcs.utils import author_name, author_email
 from rhodecode.lib import credentials_filter, age as _age
 from rhodecode.model.db import User
 
 age = lambda  x: _age(x)
 capitalize = lambda x: x.capitalize()
 email = author_email
 short_id = lambda x: x[:12]
 hide_credentials = lambda x: ''.join(credentials_filter(x))
 
 
 def is_git(repository):
     if hasattr(repository, 'alias'):
         _type = repository.alias
     elif hasattr(repository, 'repo_type'):
         _type = repository.repo_type
     else:
         _type = repository
     return _type == 'git'
 
 
 def is_hg(repository):
     if hasattr(repository, 'alias'):
         _type = repository.alias
     elif hasattr(repository, 'repo_type'):
         _type = repository.repo_type
     else:
         _type = repository
     return _type == 'hg'
 
 
 def email_or_none(author):
     _email = email(author)
     if _email != '':
         return _email
 
     # See if it contains a username we can get an email from
     user = User.get_by_username(author_name(author), case_insensitive=True,
                                 cache=True)
     if user is not None:
         return user.email
 
     # No valid email, not a valid user in the system, none!
     return None
 
 
 def person(author):
     # attr to return from fetched user
     person_getter = lambda usr: usr.username
 
     # Valid email in the attribute passed, see if they're in the system
     _email = email(author)
     if _email != '':
         user = User.get_by_email(_email, case_insensitive=True, cache=True)
         if user is not None:
             return person_getter(user)
         return _email
 
     # Maybe it's a username?
     _author = author_name(author)
     user = User.get_by_username(_author, case_insensitive=True,
                                 cache=True)
     if user is not None:
         return person_getter(user)
 
     # Still nothing?  Just pass back the author name then
     return _author
 
 
 def bool2icon(value):
     """Returns True/False values represented as small html image of true/false
     icons
 
     :param value: bool value
     """
 
     if value is True:
         return HTML.tag('img', src=url("/images/icons/accept.png"),
                         alt=_('True'))
 
     if value is False:
         return HTML.tag('img', src=url("/images/icons/cancel.png"),
                         alt=_('False'))
 
     return value
 
 
 def action_parser(user_log, feed=False):
     """This helper will action_map the specified string action into translated
     fancy names with icons and links
 
     :param user_log: user log instance
     :param feed: use output for feeds (no html and fancy icons)
     """
 
     action = user_log.action
     action_params = ' '
 
     x = action.split(':')
 
     if len(x) > 1:
         action, action_params = x
 
     def get_cs_links():
         revs_limit = 3 #display this amount always
         revs_top_limit = 50 #show upto this amount of changesets hidden
         revs = action_params.split(',')
         repo_name = user_log.repository.repo_name
 
         from rhodecode.model.scm import ScmModel
         repo = user_log.repository.scm_instance
 
         message = lambda rev: get_changeset_safe(repo, rev).message
         cs_links = []
         cs_links.append(" " + ', '.join ([link_to(rev,
                 url('changeset_home',
                 repo_name=repo_name,
                 revision=rev), title=tooltip(message(rev)),
                 class_='tooltip') for rev in revs[:revs_limit] ]))
 
         compare_view = (' <div class="compare_view tooltip" title="%s">'
                         '<a href="%s">%s</a> '
                         '</div>' % (_('Show all combined changesets %s->%s' \
                                       % (revs[0], revs[-1])),
                                     url('changeset_home', repo_name=repo_name,
                                         revision='%s...%s' % (revs[0], revs[-1])
                                     ),
                                     _('compare view'))
                         )
 
         # if we have exactly one more than normally displayed:
         # just display it, takes less space than displaying "and 1 more revisions"
         if len(revs) == revs_limit + 1:
             rev = revs[revs_limit]
             cs_links.append(", " + link_to(rev,
                 url('changeset_home',
                 repo_name=repo_name,
                 revision=rev), title=tooltip(message(rev)),
                 class_='tooltip') )
 
         # hidden-by-default ones
         if len(revs) > revs_limit + 1:
             uniq_id = revs[0]
             html_tmpl = ('<span> %s '
             '<a class="show_more" id="_%s" href="#more">%s</a> '
             '%s</span>')
             if not feed:
                 cs_links.append(html_tmpl % (_('and'), uniq_id, _('%s more') \
                                         % (len(revs) - revs_limit),
                                         _('revisions')))
 
             if not feed:
                 html_tmpl = '<span id="%s" style="display:none">, %s </span>'
             else:
                 html_tmpl = '<span id="%s"> %s </span>'
 
             morelinks = ', '.join([link_to(rev,
                 url('changeset_home',
                 repo_name=repo_name, revision=rev),
                 title=message(rev), class_='tooltip')
                 for rev in revs[revs_limit:revs_top_limit]])
 
             if len(revs) > revs_top_limit:
                 morelinks += ', ...'
 
             cs_links.append(html_tmpl % (uniq_id, morelinks))
         if len(revs) > 1:
             cs_links.append(compare_view)
         return ''.join(cs_links)
 
     def get_fork_name():
         repo_name = action_params
         return _('fork name ') + str(link_to(action_params, url('summary_home',
                                           repo_name=repo_name,)))
 
     action_map = {'user_deleted_repo':(_('[deleted] repository'), None),
            'user_created_repo':(_('[created] repository'), None),
            'user_created_fork':(_('[created] repository as fork'), None),
            'user_forked_repo':(_('[forked] repository'), get_fork_name),
            'user_updated_repo':(_('[updated] repository'), None),
            'admin_deleted_repo':(_('[delete] repository'), None),
            'admin_created_repo':(_('[created] repository'), None),
            'admin_forked_repo':(_('[forked] repository'), None),
            'admin_updated_repo':(_('[updated] repository'), None),
            'push':(_('[pushed] into'), get_cs_links),
            'push_local':(_('[committed via RhodeCode] into'), get_cs_links),
            'push_remote':(_('[pulled from remote] into'), get_cs_links),
            'pull':(_('[pulled] from'), None),
            'started_following_repo':(_('[started following] repository'), None),
            'stopped_following_repo':(_('[stopped following] repository'), None),
             }
 
     action_str = action_map.get(action, action)
     if feed:
         action = action_str[0].replace('[', '').replace(']', '')
     else:
         action = action_str[0].replace('[', '<span class="journal_highlight">')\
                    .replace(']', '</span>')
 
     action_params_func = lambda :""
 
     if callable(action_str[1]):
         action_params_func = action_str[1]
 
     return [literal(action), action_params_func]
 
 
 def action_parser_icon(user_log):
     action = user_log.action
     action_params = None
     x = action.split(':')
 
     if len(x) > 1:
         action, action_params = x
 
     tmpl = """<img src="%s%s" alt="%s"/>"""
     map = {'user_deleted_repo':'database_delete.png',
            'user_created_repo':'database_add.png',
            'user_created_fork':'arrow_divide.png',
            'user_forked_repo':'arrow_divide.png',
            'user_updated_repo':'database_edit.png',
            'admin_deleted_repo':'database_delete.png',
            'admin_created_repo':'database_add.png',
            'admin_forked_repo':'arrow_divide.png',
            'admin_updated_repo':'database_edit.png',
            'push':'script_add.png',
            'push_local':'script_edit.png',
            'push_remote':'connect.png',
            'pull':'down_16.png',
            'started_following_repo':'heart_add.png',
            'stopped_following_repo':'heart_delete.png',
             }
     return literal(tmpl % ((url('/images/icons/')),
                            map.get(action, action), action))
 
 
 #==============================================================================
 # PERMS
 #==============================================================================
 from rhodecode.lib.auth import HasPermissionAny, HasPermissionAll, \
 HasRepoPermissionAny, HasRepoPermissionAll
 
 
 #==============================================================================
 # GRAVATAR URL
 #==============================================================================
 
 def gravatar_url(email_address, size=30):
     if (not str2bool(config['app_conf'].get('use_gravatar')) or
         not email_address or email_address == '[e-mail]'):
         f=lambda a,l:min(l,key=lambda x:abs(x-a))
         return url("/images/user%s.png" % f(size, [14, 16, 20, 24, 30]))
 
     ssl_enabled = 'https' == request.environ.get('wsgi.url_scheme')
     default = 'identicon'
     baseurl_nossl = "http://www.gravatar.com/avatar/"
     baseurl_ssl = "https://secure.gravatar.com/avatar/"
     baseurl = baseurl_ssl if ssl_enabled else baseurl_nossl
 
     if isinstance(email_address, unicode):
         #hashlib crashes on unicode items
         email_address = safe_str(email_address)
     # construct the url
     gravatar_url = baseurl + hashlib.md5(email_address.lower()).hexdigest() + "?"
     gravatar_url += urllib.urlencode({'d':default, 's':str(size)})
 
     return gravatar_url
 
 
 #==============================================================================
 # REPO PAGER, PAGER FOR REPOSITORY
 #==============================================================================
 class RepoPage(Page):
 
     def __init__(self, collection, page=1, items_per_page=20,
                  item_count=None, url=None, **kwargs):
 
         """Create a "RepoPage" instance. special pager for paging
         repository
         """
         self._url_generator = url
 
         # Safe the kwargs class-wide so they can be used in the pager() method
         self.kwargs = kwargs
 
         # Save a reference to the collection
         self.original_collection = collection
 
         self.collection = collection
 
         # The self.page is the number of the current page.
         # The first page has the number 1!
         try:
             self.page = int(page) # make it int() if we get it as a string
         except (ValueError, TypeError):
             self.page = 1
 
         self.items_per_page = items_per_page
 
         # Unless the user tells us how many items the collections has
         # we calculate that ourselves.
         if item_count is not None:
             self.item_count = item_count
         else:
             self.item_count = len(self.collection)
 
         # Compute the number of the first and last available page
         if self.item_count > 0:
             self.first_page = 1
             self.page_count = int(math.ceil(float(self.item_count) /
                                             self.items_per_page))
             self.last_page = self.first_page + self.page_count - 1
 
             # Make sure that the requested page number is the range of
             # valid pages
             if self.page > self.last_page:
                 self.page = self.last_page
             elif self.page < self.first_page:
                 self.page = self.first_page
 
             # Note: the number of items on this page can be less than
             #       items_per_page if the last page is not full
             self.first_item = max(0, (self.item_count) - (self.page *
                                                           items_per_page))
             self.last_item = ((self.item_count - 1) - items_per_page *
                               (self.page - 1))
 
             self.items = list(self.collection[self.first_item:self.last_item + 1])
 
             # Links to previous and next page
             if self.page > self.first_page:
                 self.previous_page = self.page - 1
             else:
                 self.previous_page = None
 
             if self.page < self.last_page:
                 self.next_page = self.page + 1
             else:
                 self.next_page = None
 
         # No items available
         else:
             self.first_page = None
             self.page_count = 0
             self.last_page = None
             self.first_item = None
             self.last_item = None
             self.previous_page = None
             self.next_page = None
             self.items = []
 
         # This is a subclass of the 'list' type. Initialise the list now.
         list.__init__(self, reversed(self.items))
 
 
 def changed_tooltip(nodes):
     """
     Generates a html string for changed nodes in changeset page.
     It limits the output to 30 entries
 
     :param nodes: LazyNodesGenerator
     """
     if nodes:
         pref = ': <br/> '
         suf = ''
         if len(nodes) > 30:
             suf = '<br/>' + _(' and %s more') % (len(nodes) - 30)
         return literal(pref + '<br/> '.join([safe_unicode(x.path)
                                              for x in nodes[:30]]) + suf)
     else:
         return ': ' + _('No Files')
 
 
 def repo_link(groups_and_repos):
     """
     Makes a breadcrumbs link to repo within a group
     joins &raquo; on each group to create a fancy link
 
     ex::
         group >> subgroup >> repo
 
     :param groups_and_repos:
     """
     groups, repo_name = groups_and_repos
 
     if not groups:
         return repo_name
     else:
         def make_link(group):
             return link_to(group.name, url('repos_group_home',
                                            group_name=group.group_name))
         return literal(' &raquo; '.join(map(make_link, groups)) + \
                        " &raquo; " + repo_name)
 
 
 def fancy_file_stats(stats):
     """
     Displays a fancy two colored bar for number of added/deleted
     lines of code on file
 
     :param stats: two element list of added/deleted lines of code
     """
 
     a, d, t = stats[0], stats[1], stats[0] + stats[1]
     width = 100
     unit = float(width) / (t or 1)
 
     # needs > 9% of width to be visible or 0 to be hidden
     a_p = max(9, unit * a) if a > 0 else 0
     d_p = max(9, unit * d) if d > 0 else 0
     p_sum = a_p + d_p
 
     if p_sum > width:
         #adjust the percentage to be == 100% since we adjusted to 9
         if a_p > d_p:
             a_p = a_p - (p_sum - width)
         else:
             d_p = d_p - (p_sum - width)
 
     a_v = a if a > 0 else ''
     d_v = d if d > 0 else ''
 
     def cgen(l_type):
         mapping = {'tr': 'top-right-rounded-corner',
                    'tl': 'top-left-rounded-corner',
                    'br': 'bottom-right-rounded-corner',
                    'bl': 'bottom-left-rounded-corner'}
         map_getter = lambda x: mapping[x]
 
         if l_type == 'a' and d_v:
             #case when added and deleted are present
             return ' '.join(map(map_getter, ['tl', 'bl']))
 
         if l_type == 'a' and not d_v:
             return ' '.join(map(map_getter, ['tr', 'br', 'tl', 'bl']))
 
         if l_type == 'd' and a_v:
             return ' '.join(map(map_getter, ['tr', 'br']))
 
         if l_type == 'd' and not a_v:
             return ' '.join(map(map_getter, ['tr', 'br', 'tl', 'bl']))
 
     d_a = '<div class="added %s" style="width:%s%%">%s</div>' % (
         cgen('a'),a_p, a_v
     )
     d_d = '<div class="deleted %s" style="width:%s%%">%s</div>' % (
         cgen('d'),d_p, d_v
     )
     return literal('<div style="width:%spx">%s%s</div>' % (width, d_a, d_d))
 
 
 def urlify_text(text_):
     import re
 
     url_pat = re.compile(r'''(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]'''
                          '''|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)''')
 
     def url_func(match_obj):
         url_full = match_obj.groups()[0]
         return '<a href="%(url)s">%(url)s</a>' % ({'url': url_full})
 
     return literal(url_pat.sub(url_func, text_))
 
 
 def urlify_changesets(text_, repository):
     import re
     URL_PAT = re.compile(r'([0-9a-fA-F]{12,})')
 
     def url_func(match_obj):
         rev = match_obj.groups()[0]
         pref = ''
         if match_obj.group().startswith(' '):
             pref = ' '
         tmpl = (
         '%(pref)s<a class="%(cls)s" href="%(url)s">'
         '%(rev)s'
         '</a>'
         )
         return tmpl % {
          'pref': pref,
          'cls': 'revision-link',
          'url': url('changeset_home', repo_name=repository, revision=rev),
          'rev': rev,
         }
 
     newtext = URL_PAT.sub(url_func, text_)
 
     return newtext
 
 
 def urlify_commit(text_, repository=None, link_=None):
     import re
     import traceback
 
     # urlify changesets
     text_ = urlify_changesets(text_, repository)
 
     def linkify_others(t,l):
         urls = re.compile(r'(\<a.*?\<\/a\>)',)
         links = []
         for e in urls.split(t):
             if not urls.match(e):
                 links.append('<a class="message-link" href="%s">%s</a>' % (l,e))
             else:
                 links.append(e)
 
         return ''.join(links)
     try:
         conf = config['app_conf']
 
         URL_PAT = re.compile(r'%s' % conf.get('issue_pat'))
 
         if URL_PAT:
             ISSUE_SERVER_LNK = conf.get('issue_server_link')
             ISSUE_PREFIX = conf.get('issue_prefix')
 
             def url_func(match_obj):
                 pref = ''
                 if match_obj.group().startswith(' '):
                     pref = ' '
 
                 issue_id = ''.join(match_obj.groups())
                 tmpl = (
                 '%(pref)s<a class="%(cls)s" href="%(url)s">'
                 '%(issue-prefix)s%(id-repr)s'
                 '</a>'
                 )
                 url = ISSUE_SERVER_LNK.replace('{id}', issue_id)
                 if repository:
                     url = url.replace('{repo}', repository)
 
                 return tmpl % {
                      'pref': pref,
                      'cls': 'issue-tracker-link',
                      'url': url,
                      'id-repr': issue_id,
                      'issue-prefix': ISSUE_PREFIX,
                      'serv': ISSUE_SERVER_LNK,
                 }
 
             newtext = URL_PAT.sub(url_func, text_)
 
             # wrap not links into final link => link_
             newtext = linkify_others(newtext, link_)
 
             return literal(newtext)
     except:
         log.error(traceback.format_exc())
         pass
 
     return text_
 
 
 def rst(source):
     return literal('<div class="rst-block">%s</div>' %
                    MarkupRenderer.rst(source))
 
 
 def rst_w_mentions(source):
     """
     Wrapped rst renderer with @mention highlighting
 
     :param source:
     """
     return literal('<div class="rst-block">%s</div>' %
                    MarkupRenderer.rst_with_mentions(source))

rhodecode/lib/indexers/daemon.py

@@ -1,235 +1,235 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.indexers.daemon
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     A daemon will read from task table and run tasks
 
     :created_on: Jan 26, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import sys
 import logging
 import traceback
 
 from shutil import rmtree
 from time import mktime
 
 from os.path import dirname as dn
 from os.path import join as jn
 
 #to get the rhodecode import
 project_path = dn(dn(dn(dn(os.path.realpath(__file__)))))
 sys.path.append(project_path)
 
 
 from rhodecode.model.scm import ScmModel
 from rhodecode.lib import safe_unicode
 from rhodecode.lib.indexers import INDEX_EXTENSIONS, SCHEMA, IDX_NAME
 
-from vcs.exceptions import ChangesetError, RepositoryError, \
+from rhodecode.lib.vcs.exceptions import ChangesetError, RepositoryError, \
     NodeDoesNotExistError
 
 from whoosh.index import create_in, open_dir
 
 
 log = logging.getLogger('whooshIndexer')
 # create logger
 log.setLevel(logging.DEBUG)
 log.propagate = False
 # create console handler and set level to debug
 ch = logging.StreamHandler()
 ch.setLevel(logging.DEBUG)
 
 # create formatter
 formatter = logging.Formatter("%(asctime)s - %(name)s -"
                               " %(levelname)s - %(message)s")
 
 # add formatter to ch
 ch.setFormatter(formatter)
 
 # add ch to logger
 log.addHandler(ch)
 
 
 class WhooshIndexingDaemon(object):
     """
     Daemon for atomic jobs
     """
 
     def __init__(self, indexname=IDX_NAME, index_location=None,
                  repo_location=None, sa=None, repo_list=None):
         self.indexname = indexname
 
         self.index_location = index_location
         if not index_location:
             raise Exception('You have to provide index location')
 
         self.repo_location = repo_location
         if not repo_location:
             raise Exception('You have to provide repositories location')
 
         self.repo_paths = ScmModel(sa).repo_scan(self.repo_location)
 
         if repo_list:
             filtered_repo_paths = {}
             for repo_name, repo in self.repo_paths.items():
                 if repo_name in repo_list:
                     filtered_repo_paths[repo_name] = repo
 
             self.repo_paths = filtered_repo_paths
 
         self.initial = False
         if not os.path.isdir(self.index_location):
             os.makedirs(self.index_location)
             log.info('Cannot run incremental index since it does not'
                      ' yet exist running full build')
             self.initial = True
 
     def get_paths(self, repo):
         """recursive walk in root dir and return a set of all path in that dir
         based on repository walk function
         """
         index_paths_ = set()
         try:
             tip = repo.get_changeset('tip')
             for topnode, dirs, files in tip.walk('/'):
                 for f in files:
                     index_paths_.add(jn(repo.path, f.path))
 
         except RepositoryError, e:
             log.debug(traceback.format_exc())
             pass
         return index_paths_
 
     def get_node(self, repo, path):
         n_path = path[len(repo.path) + 1:]
         node = repo.get_changeset().get_node(n_path)
         return node
 
     def get_node_mtime(self, node):
         return mktime(node.last_changeset.date.timetuple())
 
     def add_doc(self, writer, path, repo, repo_name):
         """Adding doc to writer this function itself fetches data from
         the instance of vcs backend"""
         node = self.get_node(repo, path)
 
         #we just index the content of chosen files, and skip binary files
         if node.extension in INDEX_EXTENSIONS and not node.is_binary:
 
             u_content = node.content
             if not isinstance(u_content, unicode):
                 log.warning('  >> %s Could not get this content as unicode '
                           'replacing with empty content', path)
                 u_content = u''
             else:
                 log.debug('    >> %s [WITH CONTENT]' % path)
 
         else:
             log.debug('    >> %s' % path)
             #just index file name without it's content
             u_content = u''
 
         writer.add_document(owner=unicode(repo.contact),
                         repository=safe_unicode(repo_name),
                         path=safe_unicode(path),
                         content=u_content,
                         modtime=self.get_node_mtime(node),
                         extension=node.extension)
 
     def build_index(self):
         if os.path.exists(self.index_location):
             log.debug('removing previous index')
             rmtree(self.index_location)
 
         if not os.path.exists(self.index_location):
             os.mkdir(self.index_location)
 
         idx = create_in(self.index_location, SCHEMA, indexname=IDX_NAME)
         writer = idx.writer()
 
         for repo_name, repo in self.repo_paths.items():
             log.debug('building index @ %s' % repo.path)
 
             for idx_path in self.get_paths(repo):
                 self.add_doc(writer, idx_path, repo, repo_name)
 
         log.debug('>> COMMITING CHANGES <<')
         writer.commit(merge=True)
         log.debug('>>> FINISHED BUILDING INDEX <<<')
 
     def update_index(self):
         log.debug('STARTING INCREMENTAL INDEXING UPDATE')
 
         idx = open_dir(self.index_location, indexname=self.indexname)
         # The set of all paths in the index
         indexed_paths = set()
         # The set of all paths we need to re-index
         to_index = set()
 
         reader = idx.reader()
         writer = idx.writer()
 
         # Loop over the stored fields in the index
         for fields in reader.all_stored_fields():
             indexed_path = fields['path']
             indexed_paths.add(indexed_path)
 
             repo = self.repo_paths[fields['repository']]
 
             try:
                 node = self.get_node(repo, indexed_path)
             except (ChangesetError, NodeDoesNotExistError):
                 # This file was deleted since it was indexed
                 log.debug('removing from index %s' % indexed_path)
                 writer.delete_by_term('path', indexed_path)
 
             else:
                 # Check if this file was changed since it was indexed
                 indexed_time = fields['modtime']
                 mtime = self.get_node_mtime(node)
                 if mtime > indexed_time:
                     # The file has changed, delete it and add it to the list of
                     # files to reindex
                     log.debug('adding to reindex list %s' % indexed_path)
                     writer.delete_by_term('path', indexed_path)
                     to_index.add(indexed_path)
 
         # Loop over the files in the filesystem
         # Assume we have a function that gathers the filenames of the
         # documents to be indexed
         for repo_name, repo in self.repo_paths.items():
             for path in self.get_paths(repo):
                 if path in to_index or path not in indexed_paths:
                     # This is either a file that's changed, or a new file
                     # that wasn't indexed before. So index it!
                     self.add_doc(writer, path, repo, repo_name)
                     log.debug('re indexing %s' % path)
 
         log.debug('>> COMMITING CHANGES <<')
         writer.commit(merge=True)
         log.debug('>>> FINISHED REBUILDING INDEX <<<')
 
     def run(self, full_index=False):
         """Run daemon"""
         if full_index or self.initial:
             self.build_index()
         else:
             self.update_index()

rhodecode/lib/middleware/simplegit.py

@@ -1,248 +1,249 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.middleware.simplegit
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     SimpleGit middleware for handling git protocol request (push/clone etc.)
     It's implemented with basic auth function
 
     :created_on: Apr 28, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import logging
 import traceback
 
 from dulwich import server as dulserver
 
+
 class SimpleGitUploadPackHandler(dulserver.UploadPackHandler):
 
     def handle(self):
         write = lambda x: self.proto.write_sideband(1, x)
 
         graph_walker = dulserver.ProtocolGraphWalker(self,
                                                      self.repo.object_store,
                                                      self.repo.get_peeled)
         objects_iter = self.repo.fetch_objects(
           graph_walker.determine_wants, graph_walker, self.progress,
           get_tagged=self.get_tagged)
 
         # Do they want any objects?
         if objects_iter is None or len(objects_iter) == 0:
             return
 
         self.progress("counting objects: %d, done.\n" % len(objects_iter))
         dulserver.write_pack_objects(dulserver.ProtocolFile(None, write),
                                   objects_iter, len(objects_iter))
         messages = []
         messages.append('thank you for using rhodecode')
 
         for msg in messages:
             self.progress(msg + "\n")
         # we are done
         self.proto.write("0000")
 
 dulserver.DEFAULT_HANDLERS = {
   'git-upload-pack': SimpleGitUploadPackHandler,
   'git-receive-pack': dulserver.ReceivePackHandler,
 }
 
 from dulwich.repo import Repo
 from dulwich.web import HTTPGitApplication
 
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 
 from rhodecode.lib import safe_str
 from rhodecode.lib.base import BaseVCSController
 from rhodecode.lib.auth import get_container_username
 from rhodecode.lib.utils import is_valid_repo
 from rhodecode.model.db import User
 
 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError
 
 log = logging.getLogger(__name__)
 
 
 def is_git(environ):
     """Returns True if request's target is git server.
     ``HTTP_USER_AGENT`` would then have git client version given.
 
     :param environ:
     """
     http_user_agent = environ.get('HTTP_USER_AGENT')
     if http_user_agent and http_user_agent.startswith('git'):
         return True
     return False
 
 
 class SimpleGit(BaseVCSController):
 
     def _handle_request(self, environ, start_response):
         if not is_git(environ):
             return self.application(environ, start_response)
 
         proxy_key = 'HTTP_X_REAL_IP'
         def_key = 'REMOTE_ADDR'
         ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
         username = None
         # skip passing error to error controller
         environ['pylons.status_code_redirect'] = True
 
         #======================================================================
         # EXTRACT REPOSITORY NAME FROM ENV
         #======================================================================
         try:
             repo_name = self.__get_repository(environ)
             log.debug('Extracted repo name is %s' % repo_name)
         except:
             return HTTPInternalServerError()(environ, start_response)
 
         #======================================================================
         # GET ACTION PULL or PUSH
         #======================================================================
         action = self.__get_action(environ)
 
         #======================================================================
         # CHECK ANONYMOUS PERMISSION
         #======================================================================
         if action in ['pull', 'push']:
             anonymous_user = self.__get_user('default')
             username = anonymous_user.username
             anonymous_perm = self._check_permission(action,anonymous_user,
                                                     repo_name)
 
             if anonymous_perm is not True or anonymous_user.active is False:
                 if anonymous_perm is not True:
                     log.debug('Not enough credentials to access this '
                               'repository as anonymous user')
                 if anonymous_user.active is False:
                     log.debug('Anonymous access is disabled, running '
                               'authentication')
                 #==============================================================
                 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
                 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
                 #==============================================================
 
                 # Attempting to retrieve username from the container
                 username = get_container_username(environ, self.config)
 
                 # If not authenticated by the container, running basic auth
                 if not username:
                     self.authenticate.realm = \
                         safe_str(self.config['rhodecode_realm'])
                     result = self.authenticate(environ)
                     if isinstance(result, str):
                         AUTH_TYPE.update(environ, 'basic')
                         REMOTE_USER.update(environ, result)
                         username = result
                     else:
                         return result.wsgi_application(environ, start_response)
 
                 #==============================================================
                 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
                 #==============================================================
 
                 if action in ['pull', 'push']:
                     try:
                         user = self.__get_user(username)
                         if user is None or not user.active:
                             return HTTPForbidden()(environ, start_response)
                         username = user.username
                     except:
                         log.error(traceback.format_exc())
                         return HTTPInternalServerError()(environ,
                                                          start_response)
 
                     #check permissions for this repository
                     perm = self._check_permission(action, user,
                                                    repo_name)
                     if perm is not True:
                         return HTTPForbidden()(environ, start_response)
 
         #===================================================================
         # GIT REQUEST HANDLING
         #===================================================================
 
         repo_path = safe_str(os.path.join(self.basepath, repo_name))
         log.debug('Repository path is %s' % repo_path)
 
         # quick check if that dir exists...
         if is_valid_repo(repo_name, self.basepath) is False:
             return HTTPNotFound()(environ, start_response)
 
         try:
             #invalidate cache on push
             if action == 'push':
                 self._invalidate_cache(repo_name)
 
             app = self.__make_app(repo_name, repo_path)
             return app(environ, start_response)
         except Exception:
             log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
 
     def __make_app(self, repo_name, repo_path):
         """
         Make an wsgi application using dulserver
 
         :param repo_name: name of the repository
         :param repo_path: full path to the repository
         """
 
         _d = {'/' + repo_name: Repo(repo_path)}
         backend = dulserver.DictBackend(_d)
         gitserve = HTTPGitApplication(backend)
 
         return gitserve
 
     def __get_repository(self, environ):
         """
         Get's repository name out of PATH_INFO header
 
         :param environ: environ where PATH_INFO is stored
         """
         try:
             environ['PATH_INFO'] = self._get_by_id(environ['PATH_INFO'])
             repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
             if repo_name.endswith('/'):
                 repo_name = repo_name.rstrip('/')
         except:
             log.error(traceback.format_exc())
             raise
         repo_name = repo_name.split('/')[0]
         return repo_name
 
     def __get_user(self, username):
         return User.get_by_username(username)
 
     def __get_action(self, environ):
         """Maps git request commands into a pull or push command.
 
         :param environ:
         """
         service = environ['QUERY_STRING'].split('=')
         if len(service) > 1:
             service_cmd = service[1]
             mapping = {'git-receive-pack': 'push',
                        'git-upload-pack': 'pull',
                        }
 
             return mapping.get(service_cmd,
                                service_cmd if service_cmd else 'other')
         else:
             return 'other'

rhodecode/lib/middleware/simplehg.py

@@ -1,251 +1,249 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.middleware.simplehg
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     SimpleHG middleware for handling mercurial protocol request
     (push/clone etc.). It's implemented with basic auth function
 
     :created_on: Apr 28, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import logging
 import traceback
 
 from mercurial.error import RepoError
 from mercurial.hgweb import hgweb_mod
 
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 
 from rhodecode.lib import safe_str
 from rhodecode.lib.base import BaseVCSController
 from rhodecode.lib.auth import get_container_username
 from rhodecode.lib.utils import make_ui, is_valid_repo, ui_sections
 from rhodecode.model.db import User
 
 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError
 
 log = logging.getLogger(__name__)
 
 
 def is_mercurial(environ):
     """Returns True if request's target is mercurial server - header
     ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
     """
     http_accept = environ.get('HTTP_ACCEPT')
     if http_accept and http_accept.startswith('application/mercurial'):
         return True
     return False
 
 
 class SimpleHg(BaseVCSController):
 
     def _handle_request(self, environ, start_response):
         if not is_mercurial(environ):
             return self.application(environ, start_response)
 
         proxy_key = 'HTTP_X_REAL_IP'
         def_key = 'REMOTE_ADDR'
         ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
 
         # skip passing error to error controller
         environ['pylons.status_code_redirect'] = True
 
         #======================================================================
         # EXTRACT REPOSITORY NAME FROM ENV
         #======================================================================
         try:
             repo_name = environ['REPO_NAME'] = self.__get_repository(environ)
             log.debug('Extracted repo name is %s' % repo_name)
         except:
             return HTTPInternalServerError()(environ, start_response)
 
         #======================================================================
         # GET ACTION PULL or PUSH
         #======================================================================
         action = self.__get_action(environ)
 
         #======================================================================
         # CHECK ANONYMOUS PERMISSION
         #======================================================================
         if action in ['pull', 'push']:
             anonymous_user = self.__get_user('default')
 
             username = anonymous_user.username
             anonymous_perm = self._check_permission(action,anonymous_user,
                                                     repo_name)
 
             if anonymous_perm is not True or anonymous_user.active is False:
                 if anonymous_perm is not True:
                     log.debug('Not enough credentials to access this '
                               'repository as anonymous user')
                 if anonymous_user.active is False:
                     log.debug('Anonymous access is disabled, running '
                               'authentication')
                 #==============================================================
                 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
                 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
                 #==============================================================
 
                 # Attempting to retrieve username from the container
                 username = get_container_username(environ, self.config)
 
                 # If not authenticated by the container, running basic auth
                 if not username:
                     self.authenticate.realm = \
                         safe_str(self.config['rhodecode_realm'])
                     result = self.authenticate(environ)
                     if isinstance(result, str):
                         AUTH_TYPE.update(environ, 'basic')
                         REMOTE_USER.update(environ, result)
                         username = result
                     else:
                         return result.wsgi_application(environ, start_response)
 
                 #==============================================================
                 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
                 #==============================================================
 
                 if action in ['pull', 'push']:
                     try:
                         user = self.__get_user(username)
                         if user is None or not user.active:
                             return HTTPForbidden()(environ, start_response)
                         username = user.username
                     except:
                         log.error(traceback.format_exc())
                         return HTTPInternalServerError()(environ,
                                                          start_response)
 
                     #check permissions for this repository
                     perm = self._check_permission(action, user,
                                                    repo_name)
                     if perm is not True:
                         return HTTPForbidden()(environ, start_response)
 
         extras = {'ip': ipaddr,
                   'username': username,
                   'action': action,
                   'repository': repo_name}
 
         #======================================================================
         # MERCURIAL REQUEST HANDLING
         #======================================================================
 
         repo_path = safe_str(os.path.join(self.basepath, repo_name))
         log.debug('Repository path is %s' % repo_path)
 
         baseui = make_ui('db')
         self.__inject_extras(repo_path, baseui, extras)
 
-
         # quick check if that dir exists...
         if is_valid_repo(repo_name, self.basepath) is False:
             return HTTPNotFound()(environ, start_response)
 
         try:
             # invalidate cache on push
             if action == 'push':
                 self._invalidate_cache(repo_name)
 
             app = self.__make_app(repo_path, baseui, extras)
             return app(environ, start_response)
         except RepoError, e:
             if str(e).find('not found') != -1:
                 return HTTPNotFound()(environ, start_response)
         except Exception:
             log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
 
     def __make_app(self, repo_name, baseui, extras):
         """
         Make an wsgi application using hgweb, and inject generated baseui
         instance, additionally inject some extras into ui object
         """
         return hgweb_mod.hgweb(repo_name, name=repo_name, baseui=baseui)
 
     def __get_repository(self, environ):
         """
         Get's repository name out of PATH_INFO header
 
         :param environ: environ where PATH_INFO is stored
         """
         try:
             environ['PATH_INFO'] = self._get_by_id(environ['PATH_INFO'])
             repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
             if repo_name.endswith('/'):
                 repo_name = repo_name.rstrip('/')
         except:
             log.error(traceback.format_exc())
             raise
 
         return repo_name
 
     def __get_user(self, username):
         return User.get_by_username(username)
 
     def __get_action(self, environ):
         """
         Maps mercurial request commands into a clone,pull or push command.
         This should always return a valid command string
 
         :param environ:
         """
         mapping = {'changegroup': 'pull',
                    'changegroupsubset': 'pull',
                    'stream_out': 'pull',
                    'listkeys': 'pull',
                    'unbundle': 'push',
                    'pushkey': 'push', }
         for qry in environ['QUERY_STRING'].split('&'):
             if qry.startswith('cmd'):
                 cmd = qry.split('=')[-1]
                 if cmd in mapping:
                     return mapping[cmd]
                 else:
                     return 'pull'
 
-
     def __inject_extras(self, repo_path, baseui, extras={}):
         """
         Injects some extra params into baseui instance
 
         also overwrites global settings with those takes from local hgrc file
 
         :param baseui: baseui instance
         :param extras: dict with extra params to put into baseui
         """
 
         hgrc = os.path.join(repo_path, '.hg', 'hgrc')
 
         # make our hgweb quiet so it doesn't print output
         baseui.setconfig('ui', 'quiet', 'true')
 
         #inject some additional parameters that will be available in ui
         #for hooks
         for k, v in extras.items():
             baseui.setconfig('rhodecode_extras', k, v)
 
         repoui = make_ui('file', hgrc, False)
 
         if repoui:
             #overwrite our ui instance with the section from hgrc file
             for section in ui_sections:
                 for k, v in repoui.configitems(section):
                     baseui.setconfig(section, k, v)

rhodecode/lib/utils.py

@@ -1,633 +1,633 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.utils
     ~~~~~~~~~~~~~~~~~~~
 
     Utilities library for RhodeCode
 
     :created_on: Apr 18, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import logging
 import datetime
 import traceback
 import paste
 import beaker
 import tarfile
 import shutil
 from os.path import abspath
 from os.path import dirname as dn, join as jn
 
 from paste.script.command import Command, BadCommand
 
 from mercurial import ui, config
 
 from webhelpers.text import collapse, remove_formatting, strip_tags
 
-from vcs import get_backend
-from vcs.backends.base import BaseChangeset
-from vcs.utils.lazy import LazyProperty
-from vcs.utils.helpers import get_scm
-from vcs.exceptions import VCSError
+from rhodecode.lib.vcs import get_backend
+from rhodecode.lib.vcs.backends.base import BaseChangeset
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs.utils.helpers import get_scm
+from rhodecode.lib.vcs.exceptions import VCSError
 
 from rhodecode.lib.caching_query import FromCache
 
 from rhodecode.model import meta
 from rhodecode.model.db import Repository, User, RhodeCodeUi, \
     UserLog, RepoGroup, RhodeCodeSetting, UserRepoGroupToPerm
 from rhodecode.model.meta import Session
 from rhodecode.model.repos_group import ReposGroupModel
 
 log = logging.getLogger(__name__)
 
 
 def recursive_replace(str_, replace=' '):
     """Recursive replace of given sign to just one instance
 
     :param str_: given string
     :param replace: char to find and replace multiple instances
 
     Examples::
     >>> recursive_replace("Mighty---Mighty-Bo--sstones",'-')
     'Mighty-Mighty-Bo-sstones'
     """
 
     if str_.find(replace * 2) == -1:
         return str_
     else:
         str_ = str_.replace(replace * 2, replace)
         return recursive_replace(str_, replace)
 
 
 def repo_name_slug(value):
     """Return slug of name of repository
     This function is called on each creation/modification
     of repository to prevent bad names in repo
     """
 
     slug = remove_formatting(value)
     slug = strip_tags(slug)
 
     for c in """=[]\;'"<>,/~!@#$%^&*()+{}|: """:
         slug = slug.replace(c, '-')
     slug = recursive_replace(slug, '-')
     slug = collapse(slug, '-')
     return slug
 
 
 def get_repo_slug(request):
     return request.environ['pylons.routes_dict'].get('repo_name')
 
 
 def get_repos_group_slug(request):
     return request.environ['pylons.routes_dict'].get('group_name')
 
 
 def action_logger(user, action, repo, ipaddr='', sa=None, commit=False):
     """
     Action logger for various actions made by users
 
     :param user: user that made this action, can be a unique username string or
         object containing user_id attribute
     :param action: action to log, should be on of predefined unique actions for
         easy translations
     :param repo: string name of repository or object containing repo_id,
         that action was made on
     :param ipaddr: optional ip address from what the action was made
     :param sa: optional sqlalchemy session
 
     """
 
     if not sa:
         sa = meta.Session
 
     try:
         if hasattr(user, 'user_id'):
             user_obj = user
         elif isinstance(user, basestring):
             user_obj = User.get_by_username(user)
         else:
             raise Exception('You have to provide user object or username')
 
         if hasattr(repo, 'repo_id'):
             repo_obj = Repository.get(repo.repo_id)
             repo_name = repo_obj.repo_name
         elif  isinstance(repo, basestring):
             repo_name = repo.lstrip('/')
             repo_obj = Repository.get_by_repo_name(repo_name)
         else:
             raise Exception('You have to provide repository to action logger')
 
         user_log = UserLog()
         user_log.user_id = user_obj.user_id
         user_log.action = action
 
         user_log.repository_id = repo_obj.repo_id
         user_log.repository_name = repo_name
 
         user_log.action_date = datetime.datetime.now()
         user_log.user_ip = ipaddr
         sa.add(user_log)
 
         log.info('Adding user %s, action %s on %s' % (user_obj, action, repo))
         if commit:
             sa.commit()
     except:
         log.error(traceback.format_exc())
         raise
 
 
 def get_repos(path, recursive=False):
     """
     Scans given path for repos and return (name,(type,path)) tuple
 
     :param path: path to scan for repositories
     :param recursive: recursive search and return names with subdirs in front
     """
 
     # remove ending slash for better results
     path = path.rstrip(os.sep)
 
     def _get_repos(p):
         if not os.access(p, os.W_OK):
             return
         for dirpath in os.listdir(p):
             if os.path.isfile(os.path.join(p, dirpath)):
                 continue
             cur_path = os.path.join(p, dirpath)
             try:
                 scm_info = get_scm(cur_path)
                 yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info
             except VCSError:
                 if not recursive:
                     continue
                 #check if this dir containts other repos for recursive scan
                 rec_path = os.path.join(p, dirpath)
                 if os.path.isdir(rec_path):
                     for inner_scm in _get_repos(rec_path):
                         yield inner_scm
 
     return _get_repos(path)
 
 
 def is_valid_repo(repo_name, base_path):
     """
     Returns True if given path is a valid repository False otherwise
     :param repo_name:
     :param base_path:
 
     :return True: if given path is a valid repository
     """
     full_path = os.path.join(base_path, repo_name)
 
     try:
         get_scm(full_path)
         return True
     except VCSError:
         return False
 
 
 def is_valid_repos_group(repos_group_name, base_path):
     """
     Returns True if given path is a repos group False otherwise
 
     :param repo_name:
     :param base_path:
     """
     full_path = os.path.join(base_path, repos_group_name)
 
     # check if it's not a repo
     if is_valid_repo(repos_group_name, base_path):
         return False
 
     # check if it's a valid path
     if os.path.isdir(full_path):
         return True
 
     return False
 
 
 def ask_ok(prompt, retries=4, complaint='Yes or no, please!'):
     while True:
         ok = raw_input(prompt)
         if ok in ('y', 'ye', 'yes'):
             return True
         if ok in ('n', 'no', 'nop', 'nope'):
             return False
         retries = retries - 1
         if retries < 0:
             raise IOError
         print complaint
 
 #propagated from mercurial documentation
 ui_sections = ['alias', 'auth',
                 'decode/encode', 'defaults',
                 'diff', 'email',
                 'extensions', 'format',
                 'merge-patterns', 'merge-tools',
                 'hooks', 'http_proxy',
                 'smtp', 'patch',
                 'paths', 'profiling',
                 'server', 'trusted',
                 'ui', 'web', ]
 
 
 def make_ui(read_from='file', path=None, checkpaths=True):
     """A function that will read python rc files or database
     and make an mercurial ui object from read options
 
     :param path: path to mercurial config file
     :param checkpaths: check the path
     :param read_from: read from 'file' or 'db'
     """
 
     baseui = ui.ui()
 
     # clean the baseui object
     baseui._ocfg = config.config()
     baseui._ucfg = config.config()
     baseui._tcfg = config.config()
 
     if read_from == 'file':
         if not os.path.isfile(path):
             log.debug('hgrc file is not present at %s skipping...' % path)
             return False
         log.debug('reading hgrc from %s' % path)
         cfg = config.config()
         cfg.read(path)
         for section in ui_sections:
             for k, v in cfg.items(section):
                 log.debug('settings ui from file[%s]%s:%s' % (section, k, v))
                 baseui.setconfig(section, k, v)
 
     elif read_from == 'db':
         sa = meta.Session
         ret = sa.query(RhodeCodeUi)\
             .options(FromCache("sql_cache_short", "get_hg_ui_settings"))\
             .all()
 
         hg_ui = ret
         for ui_ in hg_ui:
             if ui_.ui_active:
                 log.debug('settings ui from db[%s]%s:%s', ui_.ui_section,
                           ui_.ui_key, ui_.ui_value)
                 baseui.setconfig(ui_.ui_section, ui_.ui_key, ui_.ui_value)
 
         meta.Session.remove()
     return baseui
 
 
 def set_rhodecode_config(config):
     """
     Updates pylons config with new settings from database
 
     :param config:
     """
     hgsettings = RhodeCodeSetting.get_app_settings()
 
     for k, v in hgsettings.items():
         config[k] = v
 
 
 def invalidate_cache(cache_key, *args):
     """
     Puts cache invalidation task into db for
     further global cache invalidation
     """
 
     from rhodecode.model.scm import ScmModel
 
     if cache_key.startswith('get_repo_cached_'):
         name = cache_key.split('get_repo_cached_')[-1]
         ScmModel().mark_for_invalidation(name)
 
 
 class EmptyChangeset(BaseChangeset):
     """
     An dummy empty changeset. It's possible to pass hash when creating
     an EmptyChangeset
     """
 
     def __init__(self, cs='0' * 40, repo=None, requested_revision=None,
                  alias=None):
         self._empty_cs = cs
         self.revision = -1
         self.message = ''
         self.author = ''
         self.date = ''
         self.repository = repo
         self.requested_revision = requested_revision
         self.alias = alias
 
     @LazyProperty
     def raw_id(self):
         """
         Returns raw string identifying this changeset, useful for web
         representation.
         """
 
         return self._empty_cs
 
     @LazyProperty
     def branch(self):
         return get_backend(self.alias).DEFAULT_BRANCH_NAME
 
     @LazyProperty
     def short_id(self):
         return self.raw_id[:12]
 
     def get_file_changeset(self, path):
         return self
 
     def get_file_content(self, path):
         return u''
 
     def get_file_size(self, path):
         return 0
 
 
 def map_groups(groups):
     """
     Checks for groups existence, and creates groups structures.
     It returns last group in structure
 
     :param groups: list of groups structure
     """
     sa = meta.Session
 
     parent = None
     group = None
 
     # last element is repo in nested groups structure
     groups = groups[:-1]
     rgm = ReposGroupModel(sa)
     for lvl, group_name in enumerate(groups):
         log.debug('creating group level: %s group_name: %s' % (lvl, group_name))
         group_name = '/'.join(groups[:lvl] + [group_name])
         group = RepoGroup.get_by_group_name(group_name)
         desc = '%s group' % group_name
 
 #        # WTF that doesn't work !?
 #        if group is None:
 #            group = rgm.create(group_name, desc, parent, just_db=True)
 #            sa.commit()
 
         if group is None:
             group = RepoGroup(group_name, parent)
             group.group_description = desc
             sa.add(group)
             rgm._create_default_perms(group)
             sa.commit()
         parent = group
     return group
 
 
 def repo2db_mapper(initial_repo_list, remove_obsolete=False):
     """
     maps all repos given in initial_repo_list, non existing repositories
     are created, if remove_obsolete is True it also check for db entries
     that are not in initial_repo_list and removes them.
 
     :param initial_repo_list: list of repositories found by scanning methods
     :param remove_obsolete: check for obsolete entries in database
     """
     from rhodecode.model.repo import RepoModel
     sa = meta.Session
     rm = RepoModel()
     user = sa.query(User).filter(User.admin == True).first()
     if user is None:
         raise Exception('Missing administrative account !')
     added = []
 
     # fixup groups paths to new format on the fly. Helps with migration from
     # old rhodecode versions also set permissions if they are not present !
     # TODO: remove this in future, before release
     def_usr = User.get_by_username('default')
     for g in RepoGroup.query().all():
         g.group_name = g.get_new_name(g.name)
         sa.add(g)
         # get default perm
         default = UserRepoGroupToPerm.query()\
             .filter(UserRepoGroupToPerm.group == g)\
             .filter(UserRepoGroupToPerm.user == def_usr)\
             .scalar()
 
         if default is None:
             log.debug('missing default permission for group %s adding' % g)
             ReposGroupModel()._create_default_perms(g)
 
     for name, repo in initial_repo_list.items():
         group = map_groups(name.split(Repository.url_sep()))
         if not rm.get_by_repo_name(name, cache=False):
             log.info('repository %s not found creating default' % name)
             added.append(name)
             form_data = {
              'repo_name': name,
              'repo_name_full': name,
              'repo_type': repo.alias,
              'description': repo.description \
                 if repo.description != 'unknown' else '%s repository' % name,
              'private': False,
              'group_id': getattr(group, 'group_id', None)
             }
             rm.create(form_data, user, just_db=True)
     sa.commit()
     removed = []
     if remove_obsolete:
         #remove from database those repositories that are not in the filesystem
         for repo in sa.query(Repository).all():
             if repo.repo_name not in initial_repo_list.keys():
                 removed.append(repo.repo_name)
                 sa.delete(repo)
                 sa.commit()
 
     return added, removed
 
 
 # set cache regions for beaker so celery can utilise it
 def add_cache(settings):
     cache_settings = {'regions': None}
     for key in settings.keys():
         for prefix in ['beaker.cache.', 'cache.']:
             if key.startswith(prefix):
                 name = key.split(prefix)[1].strip()
                 cache_settings[name] = settings[key].strip()
     if cache_settings['regions']:
         for region in cache_settings['regions'].split(','):
             region = region.strip()
             region_settings = {}
             for key, value in cache_settings.items():
                 if key.startswith(region):
                     region_settings[key.split('.')[1]] = value
             region_settings['expire'] = int(region_settings.get('expire',
                                                                 60))
             region_settings.setdefault('lock_dir',
                                        cache_settings.get('lock_dir'))
             region_settings.setdefault('data_dir',
                                        cache_settings.get('data_dir'))
 
             if 'type' not in region_settings:
                 region_settings['type'] = cache_settings.get('type',
                                                              'memory')
             beaker.cache.cache_regions[region] = region_settings
 
 
 #==============================================================================
 # TEST FUNCTIONS AND CREATORS
 #==============================================================================
 def create_test_index(repo_location, config, full_index):
     """
     Makes default test index
 
     :param config: test config
     :param full_index:
     """
 
     from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
     from rhodecode.lib.pidlock import DaemonLock, LockHeld
 
     repo_location = repo_location
 
     index_location = os.path.join(config['app_conf']['index_dir'])
     if not os.path.exists(index_location):
         os.makedirs(index_location)
 
     try:
         l = DaemonLock(file_=jn(dn(index_location), 'make_index.lock'))
         WhooshIndexingDaemon(index_location=index_location,
                              repo_location=repo_location)\
             .run(full_index=full_index)
         l.release()
     except LockHeld:
         pass
 
 
 def create_test_env(repos_test_path, config):
     """
     Makes a fresh database and
     install test repository into tmp dir
     """
     from rhodecode.lib.db_manage import DbManage
     from rhodecode.tests import HG_REPO, TESTS_TMP_PATH
 
     # PART ONE create db
     dbconf = config['sqlalchemy.db1.url']
     log.debug('making test db %s' % dbconf)
 
     # create test dir if it doesn't exist
     if not os.path.isdir(repos_test_path):
         log.debug('Creating testdir %s' % repos_test_path)
         os.makedirs(repos_test_path)
 
     dbmanage = DbManage(log_sql=True, dbconf=dbconf, root=config['here'],
                         tests=True)
     dbmanage.create_tables(override=True)
     dbmanage.create_settings(dbmanage.config_prompt(repos_test_path))
     dbmanage.create_default_user()
     dbmanage.admin_prompt()
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
     Session.commit()
     # PART TWO make test repo
     log.debug('making test vcs repositories')
 
     idx_path = config['app_conf']['index_dir']
     data_path = config['app_conf']['cache_dir']
 
     #clean index and data
     if idx_path and os.path.exists(idx_path):
         log.debug('remove %s' % idx_path)
         shutil.rmtree(idx_path)
 
     if data_path and os.path.exists(data_path):
         log.debug('remove %s' % data_path)
         shutil.rmtree(data_path)
 
     #CREATE DEFAULT HG REPOSITORY
     cur_dir = dn(dn(abspath(__file__)))
     tar = tarfile.open(jn(cur_dir, 'tests', "vcs_test_hg.tar.gz"))
     tar.extractall(jn(TESTS_TMP_PATH, HG_REPO))
     tar.close()
 
 
 #==============================================================================
 # PASTER COMMANDS
 #==============================================================================
 class BasePasterCommand(Command):
     """
     Abstract Base Class for paster commands.
 
     The celery commands are somewhat aggressive about loading
     celery.conf, and since our module sets the `CELERY_LOADER`
     environment variable to our loader, we have to bootstrap a bit and
     make sure we've had a chance to load the pylons config off of the
     command line, otherwise everything fails.
     """
     min_args = 1
     min_args_error = "Please provide a paster config file as an argument."
     takes_config_file = 1
     requires_config_file = True
 
     def notify_msg(self, msg, log=False):
         """Make a notification to user, additionally if logger is passed
         it logs this action using given logger
 
         :param msg: message that will be printed to user
         :param log: logging instance, to use to additionally log this message
 
         """
         if log and isinstance(log, logging):
             log(msg)
 
     def run(self, args):
         """
         Overrides Command.run
 
         Checks for a config file argument and loads it.
         """
         if len(args) < self.min_args:
             raise BadCommand(
                 self.min_args_error % {'min_args': self.min_args,
                                        'actual_args': len(args)})
 
         # Decrement because we're going to lob off the first argument.
         # @@ This is hacky
         self.min_args -= 1
         self.bootstrap_config(args[0])
         self.update_parser()
         return super(BasePasterCommand, self).run(args[1:])
 
     def update_parser(self):
         """
         Abstract method.  Allows for the class's parser to be updated
         before the superclass's `run` method is called.  Necessary to
         allow options/arguments to be passed through to the underlying
         celery command.
         """
         raise NotImplementedError("Abstract Method.")
 
     def bootstrap_config(self, conf):
         """
         Loads the pylons configuration.
         """
         from pylons import config as pylonsconfig
 
         path_to_ini_file = os.path.realpath(conf)
         conf = paste.deploy.appconfig('config:' + path_to_ini_file)
         pylonsconfig.init_app(conf.global_conf, conf.local_conf)

rhodecode/model/db.py

@@ -1,1185 +1,1185 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.model.db
     ~~~~~~~~~~~~~~~~~~
 
     Database Models for RhodeCode
 
     :created_on: Apr 08, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import logging
 import datetime
 import traceback
 from collections import defaultdict
 
 from sqlalchemy import *
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import relationship, joinedload, class_mapper, validates
 from beaker.cache import cache_region, region_invalidate
 
-from vcs import get_backend
-from vcs.utils.helpers import get_scm
-from vcs.exceptions import VCSError
-from vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs import get_backend
+from rhodecode.lib.vcs.utils.helpers import get_scm
+from rhodecode.lib.vcs.exceptions import VCSError
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
 
 from rhodecode.lib import str2bool, safe_str, get_changeset_safe, safe_unicode
 from rhodecode.lib.compat import json
 from rhodecode.lib.caching_query import FromCache
 
 from rhodecode.model.meta import Base, Session
 
 
 log = logging.getLogger(__name__)
 
 #==============================================================================
 # BASE CLASSES
 #==============================================================================
 
 
 class ModelSerializer(json.JSONEncoder):
     """
     Simple Serializer for JSON,
 
     usage::
 
         to make object customized for serialization implement a __json__
         method that will return a dict for serialization into json
 
     example::
 
         class Task(object):
 
             def __init__(self, name, value):
                 self.name = name
                 self.value = value
 
             def __json__(self):
                 return dict(name=self.name,
                             value=self.value)
 
     """
 
     def default(self, obj):
 
         if hasattr(obj, '__json__'):
             return obj.__json__()
         else:
             return json.JSONEncoder.default(self, obj)
 
 
 class BaseModel(object):
     """
     Base Model for all classess
     """
 
     @classmethod
     def _get_keys(cls):
         """return column names for this model """
         return class_mapper(cls).c.keys()
 
     def get_dict(self):
         """
         return dict with keys and values corresponding
         to this model data """
 
         d = {}
         for k in self._get_keys():
             d[k] = getattr(self, k)
 
         # also use __json__() if present to get additional fields
         for k, val in getattr(self, '__json__', lambda: {})().iteritems():
             d[k] = val
         return d
 
     def get_appstruct(self):
         """return list with keys and values tupples corresponding
         to this model data """
 
         l = []
         for k in self._get_keys():
             l.append((k, getattr(self, k),))
         return l
 
     def populate_obj(self, populate_dict):
         """populate model with data from given populate_dict"""
 
         for k in self._get_keys():
             if k in populate_dict:
                 setattr(self, k, populate_dict[k])
 
     @classmethod
     def query(cls):
         return Session.query(cls)
 
     @classmethod
     def get(cls, id_):
         if id_:
             return cls.query().get(id_)
 
     @classmethod
     def getAll(cls):
         return cls.query().all()
 
     @classmethod
     def delete(cls, id_):
         obj = cls.query().get(id_)
         Session.delete(obj)
 
 
 class RhodeCodeSetting(Base, BaseModel):
     __tablename__ = 'rhodecode_settings'
     __table_args__ = (
         UniqueConstraint('app_settings_name'),
         {'extend_existing': True}
     )
     app_settings_id = Column("app_settings_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     app_settings_name = Column("app_settings_name", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     _app_settings_value = Column("app_settings_value", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
     def __init__(self, k='', v=''):
         self.app_settings_name = k
         self.app_settings_value = v
 
     @validates('_app_settings_value')
     def validate_settings_value(self, key, val):
         assert type(val) == unicode
         return val
 
     @hybrid_property
     def app_settings_value(self):
         v = self._app_settings_value
         if v == 'ldap_active':
             v = str2bool(v)
         return v
 
     @app_settings_value.setter
     def app_settings_value(self, val):
         """
         Setter that will always make sure we use unicode in app_settings_value
 
         :param val:
         """
         self._app_settings_value = safe_unicode(val)
 
     def __repr__(self):
         return "<%s('%s:%s')>" % (
             self.__class__.__name__,
             self.app_settings_name, self.app_settings_value
         )
 
     @classmethod
     def get_by_name(cls, ldap_key):
         return cls.query()\
             .filter(cls.app_settings_name == ldap_key).scalar()
 
     @classmethod
     def get_app_settings(cls, cache=False):
 
         ret = cls.query()
 
         if cache:
             ret = ret.options(FromCache("sql_cache_short", "get_hg_settings"))
 
         if not ret:
             raise Exception('Could not get application settings !')
         settings = {}
         for each in ret:
             settings['rhodecode_' + each.app_settings_name] = \
                 each.app_settings_value
 
         return settings
 
     @classmethod
     def get_ldap_settings(cls, cache=False):
         ret = cls.query()\
                 .filter(cls.app_settings_name.startswith('ldap_')).all()
         fd = {}
         for row in ret:
             fd.update({row.app_settings_name:row.app_settings_value})
 
         return fd
 
 
 class RhodeCodeUi(Base, BaseModel):
     __tablename__ = 'rhodecode_ui'
     __table_args__ = (
         UniqueConstraint('ui_key'),
         {'extend_existing': True}
     )
 
     HOOK_UPDATE = 'changegroup.update'
     HOOK_REPO_SIZE = 'changegroup.repo_size'
     HOOK_PUSH = 'pretxnchangegroup.push_logger'
     HOOK_PULL = 'preoutgoing.pull_logger'
 
     ui_id = Column("ui_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     ui_section = Column("ui_section", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_key = Column("ui_key", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_value = Column("ui_value", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_active = Column("ui_active", Boolean(), nullable=True, unique=None, default=True)
 
     @classmethod
     def get_by_key(cls, key):
         return cls.query().filter(cls.ui_key == key)
 
     @classmethod
     def get_builtin_hooks(cls):
         q = cls.query()
         q = q.filter(cls.ui_key.in_([cls.HOOK_UPDATE,
                                     cls.HOOK_REPO_SIZE,
                                     cls.HOOK_PUSH, cls.HOOK_PULL]))
         return q.all()
 
     @classmethod
     def get_custom_hooks(cls):
         q = cls.query()
         q = q.filter(~cls.ui_key.in_([cls.HOOK_UPDATE,
                                     cls.HOOK_REPO_SIZE,
                                     cls.HOOK_PUSH, cls.HOOK_PULL]))
         q = q.filter(cls.ui_section == 'hooks')
         return q.all()
 
     @classmethod
     def create_or_update_hook(cls, key, val):
         new_ui = cls.get_by_key(key).scalar() or cls()
         new_ui.ui_section = 'hooks'
         new_ui.ui_active = True
         new_ui.ui_key = key
         new_ui.ui_value = val
 
         Session.add(new_ui)
 
 
 class User(Base, BaseModel):
     __tablename__ = 'users'
     __table_args__ = (
         UniqueConstraint('username'), UniqueConstraint('email'),
         {'extend_existing': True}
     )
     user_id = Column("user_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     username = Column("username", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     password = Column("password", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     active = Column("active", Boolean(), nullable=True, unique=None, default=None)
     admin = Column("admin", Boolean(), nullable=True, unique=None, default=False)
     name = Column("name", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     lastname = Column("lastname", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     _email = Column("email", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     last_login = Column("last_login", DateTime(timezone=False), nullable=True, unique=None, default=None)
     ldap_dn = Column("ldap_dn", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     api_key = Column("api_key", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
     user_log = relationship('UserLog', cascade='all')
     user_perms = relationship('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id", cascade='all')
 
     repositories = relationship('Repository')
     user_followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_user_id==User.user_id', cascade='all')
     repo_to_perm = relationship('UserRepoToPerm', primaryjoin='UserRepoToPerm.user_id==User.user_id', cascade='all')
 
     group_member = relationship('UsersGroupMember', cascade='all')
 
     notifications = relationship('UserNotification',)
 
     @hybrid_property
     def email(self):
         return self._email
 
     @email.setter
     def email(self, val):
         self._email = val.lower() if val else None
 
     @property
     def full_name(self):
         return '%s %s' % (self.name, self.lastname)
 
     @property
     def full_name_or_username(self):
         return ('%s %s' % (self.name, self.lastname)
                 if (self.name and self.lastname) else self.username)
 
     @property
     def full_contact(self):
         return '%s %s <%s>' % (self.name, self.lastname, self.email)
 
     @property
     def short_contact(self):
         return '%s %s' % (self.name, self.lastname)
 
     @property
     def is_admin(self):
         return self.admin
 
     def __repr__(self):
         return "<%s('id:%s:%s')>" % (self.__class__.__name__,
                                      self.user_id, self.username)
 
     @classmethod
     def get_by_username(cls, username, case_insensitive=False, cache=False):
         if case_insensitive:
             q = cls.query().filter(cls.username.ilike(username))
         else:
             q = cls.query().filter(cls.username == username)
 
         if cache:
             q = q.options(FromCache("sql_cache_short",
                                     "get_user_%s" % username))
         return q.scalar()
 
     @classmethod
     def get_by_api_key(cls, api_key, cache=False):
         q = cls.query().filter(cls.api_key == api_key)
 
         if cache:
             q = q.options(FromCache("sql_cache_short",
                                     "get_api_key_%s" % api_key))
         return q.scalar()
 
     @classmethod
     def get_by_email(cls, email, case_insensitive=False, cache=False):
         if case_insensitive:
             q = cls.query().filter(cls.email.ilike(email))
         else:
             q = cls.query().filter(cls.email == email)
 
         if cache:
             q = q.options(FromCache("sql_cache_short",
                                     "get_api_key_%s" % email))
         return q.scalar()
 
     def update_lastlogin(self):
         """Update user lastlogin"""
         self.last_login = datetime.datetime.now()
         Session.add(self)
         log.debug('updated user %s lastlogin' % self.username)
 
     def __json__(self):
         return dict(
             email=self.email,
             full_name=self.full_name,
             full_name_or_username=self.full_name_or_username,
             short_contact=self.short_contact,
             full_contact=self.full_contact
         )
 
 
 class UserLog(Base, BaseModel):
     __tablename__ = 'user_logs'
     __table_args__ = {'extend_existing': True}
     user_log_id = Column("user_log_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=True)
     repository_name = Column("repository_name", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     user_ip = Column("user_ip", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     action = Column("action", UnicodeText(length=1200000, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     action_date = Column("action_date", DateTime(timezone=False), nullable=True, unique=None, default=None)
 
     @property
     def action_as_day(self):
         return datetime.date(*self.action_date.timetuple()[:3])
 
     user = relationship('User')
     repository = relationship('Repository',cascade='')
 
 
 class UsersGroup(Base, BaseModel):
     __tablename__ = 'users_groups'
     __table_args__ = {'extend_existing': True}
 
     users_group_id = Column("users_group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_name = Column("users_group_name", String(length=255, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
     users_group_active = Column("users_group_active", Boolean(), nullable=True, unique=None, default=None)
 
     members = relationship('UsersGroupMember', cascade="all, delete, delete-orphan", lazy="joined")
 
     def __repr__(self):
         return '<userGroup(%s)>' % (self.users_group_name)
 
     @classmethod
     def get_by_group_name(cls, group_name, cache=False,
                           case_insensitive=False):
         if case_insensitive:
             q = cls.query().filter(cls.users_group_name.ilike(group_name))
         else:
             q = cls.query().filter(cls.users_group_name == group_name)
         if cache:
             q = q.options(FromCache("sql_cache_short",
                                     "get_user_%s" % group_name))
         return q.scalar()
 
     @classmethod
     def get(cls, users_group_id, cache=False):
         users_group = cls.query()
         if cache:
             users_group = users_group.options(FromCache("sql_cache_short",
                                     "get_users_group_%s" % users_group_id))
         return users_group.get(users_group_id)
 
 
 class UsersGroupMember(Base, BaseModel):
     __tablename__ = 'users_groups_members'
     __table_args__ = {'extend_existing': True}
 
     users_group_member_id = Column("users_group_member_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User', lazy='joined')
     users_group = relationship('UsersGroup')
 
     def __init__(self, gr_id='', u_id=''):
         self.users_group_id = gr_id
         self.user_id = u_id
 
 class Repository(Base, BaseModel):
     __tablename__ = 'repositories'
     __table_args__ = (
         UniqueConstraint('repo_name'),
         {'extend_existing': True},
     )
 
     repo_id = Column("repo_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     repo_name = Column("repo_name", String(length=255, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
     clone_uri = Column("clone_uri", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=False, default=None)
     repo_type = Column("repo_type", String(length=255, convert_unicode=False, assert_unicode=None), nullable=False, unique=False, default='hg')
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=False, default=None)
     private = Column("private", Boolean(), nullable=True, unique=None, default=None)
     enable_statistics = Column("statistics", Boolean(), nullable=True, unique=None, default=True)
     enable_downloads = Column("downloads", Boolean(), nullable=True, unique=None, default=True)
     description = Column("description", String(length=10000, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     created_on = Column('created_on', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)
 
     fork_id = Column("fork_id", Integer(), ForeignKey('repositories.repo_id'), nullable=True, unique=False, default=None)
     group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=True, unique=False, default=None)
 
     user = relationship('User')
     fork = relationship('Repository', remote_side=repo_id)
     group = relationship('RepoGroup')
     repo_to_perm = relationship('UserRepoToPerm', cascade='all', order_by='UserRepoToPerm.repo_to_perm_id')
     users_group_to_perm = relationship('UsersGroupRepoToPerm', cascade='all')
     stats = relationship('Statistics', cascade='all', uselist=False)
 
     followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_repo_id==Repository.repo_id', cascade='all')
 
     logs = relationship('UserLog')
 
     def __repr__(self):
         return "<%s('%s:%s')>" % (self.__class__.__name__,
                                   self.repo_id, self.repo_name)
 
     @classmethod
     def url_sep(cls):
         return '/'
 
     @classmethod
     def get_by_repo_name(cls, repo_name):
         q = Session.query(cls).filter(cls.repo_name == repo_name)
         q = q.options(joinedload(Repository.fork))\
                 .options(joinedload(Repository.user))\
                 .options(joinedload(Repository.group))
         return q.scalar()
 
     @classmethod
     def get_repo_forks(cls, repo_id):
         return cls.query().filter(Repository.fork_id == repo_id)
 
     @classmethod
     def base_path(cls):
         """
         Returns base path when all repos are stored
 
         :param cls:
         """
         q = Session.query(RhodeCodeUi)\
             .filter(RhodeCodeUi.ui_key == cls.url_sep())
         q = q.options(FromCache("sql_cache_short", "repository_repo_path"))
         return q.one().ui_value
 
     @property
     def just_name(self):
         return self.repo_name.split(Repository.url_sep())[-1]
 
     @property
     def groups_with_parents(self):
         groups = []
         if self.group is None:
             return groups
 
         cur_gr = self.group
         groups.insert(0, cur_gr)
         while 1:
             gr = getattr(cur_gr, 'parent_group', None)
             cur_gr = cur_gr.parent_group
             if gr is None:
                 break
             groups.insert(0, gr)
 
         return groups
 
     @property
     def groups_and_repo(self):
         return self.groups_with_parents, self.just_name
 
     @LazyProperty
     def repo_path(self):
         """
         Returns base full path for that repository means where it actually
         exists on a filesystem
         """
         q = Session.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key ==
                                               Repository.url_sep())
         q = q.options(FromCache("sql_cache_short", "repository_repo_path"))
         return q.one().ui_value
 
     @property
     def repo_full_path(self):
         p = [self.repo_path]
         # we need to split the name by / since this is how we store the
         # names in the database, but that eventually needs to be converted
         # into a valid system path
         p += self.repo_name.split(Repository.url_sep())
         return os.path.join(*p)
 
     def get_new_name(self, repo_name):
         """
         returns new full repository name based on assigned group and new new
 
         :param group_name:
         """
         path_prefix = self.group.full_path_splitted if self.group else []
         return Repository.url_sep().join(path_prefix + [repo_name])
 
     @property
     def _ui(self):
         """
         Creates an db based ui object for this repository
         """
         from mercurial import ui
         from mercurial import config
         baseui = ui.ui()
 
         #clean the baseui object
         baseui._ocfg = config.config()
         baseui._ucfg = config.config()
         baseui._tcfg = config.config()
 
         ret = RhodeCodeUi.query()\
             .options(FromCache("sql_cache_short", "repository_repo_ui")).all()
 
         hg_ui = ret
         for ui_ in hg_ui:
             if ui_.ui_active:
                 log.debug('settings ui from db[%s]%s:%s', ui_.ui_section,
                           ui_.ui_key, ui_.ui_value)
                 baseui.setconfig(ui_.ui_section, ui_.ui_key, ui_.ui_value)
 
         return baseui
 
     @classmethod
     def is_valid(cls, repo_name):
         """
         returns True if given repo name is a valid filesystem repository
 
         :param cls:
         :param repo_name:
         """
         from rhodecode.lib.utils import is_valid_repo
 
         return is_valid_repo(repo_name, cls.base_path())
 
     #==========================================================================
     # SCM PROPERTIES
     #==========================================================================
 
     def get_changeset(self, rev):
         return get_changeset_safe(self.scm_instance, rev)
 
     @property
     def tip(self):
         return self.get_changeset('tip')
 
     @property
     def author(self):
         return self.tip.author
 
     @property
     def last_change(self):
         return self.scm_instance.last_change
 
     def comments(self, revisions=None):
         """
         Returns comments for this repository grouped by revisions
 
         :param revisions: filter query by revisions only
         """
         cmts = ChangesetComment.query()\
             .filter(ChangesetComment.repo == self)
         if revisions:
             cmts = cmts.filter(ChangesetComment.revision.in_(revisions))
         grouped = defaultdict(list)
         for cmt in cmts.all():
             grouped[cmt.revision].append(cmt)
         return grouped
 
     #==========================================================================
     # SCM CACHE INSTANCE
     #==========================================================================
 
     @property
     def invalidate(self):
         return CacheInvalidation.invalidate(self.repo_name)
 
     def set_invalidate(self):
         """
         set a cache for invalidation for this instance
         """
         CacheInvalidation.set_invalidate(self.repo_name)
 
     @LazyProperty
     def scm_instance(self):
         return self.__get_instance()
 
     @property
     def scm_instance_cached(self):
         @cache_region('long_term')
         def _c(repo_name):
             return self.__get_instance()
         rn = self.repo_name
         log.debug('Getting cached instance of repo')
         inv = self.invalidate
         if inv is not None:
             region_invalidate(_c, None, rn)
             # update our cache
             CacheInvalidation.set_valid(inv.cache_key)
         return _c(rn)
 
     def __get_instance(self):
         repo_full_path = self.repo_full_path
         try:
             alias = get_scm(repo_full_path)[0]
             log.debug('Creating instance of %s repository' % alias)
             backend = get_backend(alias)
         except VCSError:
             log.error(traceback.format_exc())
             log.error('Perhaps this repository is in db and not in '
                       'filesystem run rescan repositories with '
                       '"destroy old data " option from admin panel')
             return
 
         if alias == 'hg':
 
             repo = backend(safe_str(repo_full_path), create=False,
                            baseui=self._ui)
             # skip hidden web repository
             if repo._get_hidden():
                 return
         else:
             repo = backend(repo_full_path, create=False)
 
         return repo
 
 
 class RepoGroup(Base, BaseModel):
     __tablename__ = 'groups'
     __table_args__ = (
         UniqueConstraint('group_name', 'group_parent_id'),
         CheckConstraint('group_id != group_parent_id'),
         {'extend_existing': True},
     )
     __mapper_args__ = {'order_by': 'group_name'}
 
     group_id = Column("group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     group_name = Column("group_name", String(length=255, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
     group_parent_id = Column("group_parent_id", Integer(), ForeignKey('groups.group_id'), nullable=True, unique=None, default=None)
     group_description = Column("group_description", String(length=10000, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
     repo_group_to_perm = relationship('UserRepoGroupToPerm', cascade='all', order_by='UserRepoGroupToPerm.group_to_perm_id')
     users_group_to_perm = relationship('UsersGroupRepoGroupToPerm', cascade='all')
 
     parent_group = relationship('RepoGroup', remote_side=group_id)
 
     def __init__(self, group_name='', parent_group=None):
         self.group_name = group_name
         self.parent_group = parent_group
 
     def __repr__(self):
         return "<%s('%s:%s')>" % (self.__class__.__name__, self.group_id,
                                   self.group_name)
 
     @classmethod
     def groups_choices(cls):
         from webhelpers.html import literal as _literal
         repo_groups = [('', '')]
         sep = ' &raquo; '
         _name = lambda k: _literal(sep.join(k))
 
         repo_groups.extend([(x.group_id, _name(x.full_path_splitted))
                               for x in cls.query().all()])
 
         repo_groups = sorted(repo_groups, key=lambda t: t[1].split(sep)[0])
         return repo_groups
 
     @classmethod
     def url_sep(cls):
         return '/'
 
     @classmethod
     def get_by_group_name(cls, group_name, cache=False, case_insensitive=False):
         if case_insensitive:
             gr = cls.query()\
                 .filter(cls.group_name.ilike(group_name))
         else:
             gr = cls.query()\
                 .filter(cls.group_name == group_name)
         if cache:
             gr = gr.options(FromCache("sql_cache_short",
                                           "get_group_%s" % group_name))
         return gr.scalar()
 
     @property
     def parents(self):
         parents_recursion_limit = 5
         groups = []
         if self.parent_group is None:
             return groups
         cur_gr = self.parent_group
         groups.insert(0, cur_gr)
         cnt = 0
         while 1:
             cnt += 1
             gr = getattr(cur_gr, 'parent_group', None)
             cur_gr = cur_gr.parent_group
             if gr is None:
                 break
             if cnt == parents_recursion_limit:
                 # this will prevent accidental infinit loops
                 log.error('group nested more than %s' %
                           parents_recursion_limit)
                 break
 
             groups.insert(0, gr)
         return groups
 
     @property
     def children(self):
         return RepoGroup.query().filter(RepoGroup.parent_group == self)
 
     @property
     def name(self):
         return self.group_name.split(RepoGroup.url_sep())[-1]
 
     @property
     def full_path(self):
         return self.group_name
 
     @property
     def full_path_splitted(self):
         return self.group_name.split(RepoGroup.url_sep())
 
     @property
     def repositories(self):
         return Repository.query().filter(Repository.group == self)
 
     @property
     def repositories_recursive_count(self):
         cnt = self.repositories.count()
 
         def children_count(group):
             cnt = 0
             for child in group.children:
                 cnt += child.repositories.count()
                 cnt += children_count(child)
             return cnt
 
         return cnt + children_count(self)
 
     def get_new_name(self, group_name):
         """
         returns new full group name based on parent and new name
 
         :param group_name:
         """
         path_prefix = (self.parent_group.full_path_splitted if
                        self.parent_group else [])
         return RepoGroup.url_sep().join(path_prefix + [group_name])
 
 
 class Permission(Base, BaseModel):
     __tablename__ = 'permissions'
     __table_args__ = {'extend_existing': True}
     permission_id = Column("permission_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     permission_name = Column("permission_name", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     permission_longname = Column("permission_longname", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
     def __repr__(self):
         return "<%s('%s:%s')>" % (
             self.__class__.__name__, self.permission_id, self.permission_name
         )
 
     @classmethod
     def get_by_key(cls, key):
         return cls.query().filter(cls.permission_name == key).scalar()
 
     @classmethod
     def get_default_perms(cls, default_user_id):
         q = Session.query(UserRepoToPerm, Repository, cls)\
          .join((Repository, UserRepoToPerm.repository_id == Repository.repo_id))\
          .join((cls, UserRepoToPerm.permission_id == cls.permission_id))\
          .filter(UserRepoToPerm.user_id == default_user_id)
 
         return q.all()
 
     @classmethod
     def get_default_group_perms(cls, default_user_id):
         q = Session.query(UserRepoGroupToPerm, RepoGroup, cls)\
          .join((RepoGroup, UserRepoGroupToPerm.group_id == RepoGroup.group_id))\
          .join((cls, UserRepoGroupToPerm.permission_id == cls.permission_id))\
          .filter(UserRepoGroupToPerm.user_id == default_user_id)
 
         return q.all()
 
 
 class UserRepoToPerm(Base, BaseModel):
     __tablename__ = 'repo_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'repository_id', 'permission_id'),
         {'extend_existing': True}
     )
     repo_to_perm_id = Column("repo_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User')
     repository = relationship('Repository')
     permission = relationship('Permission')
 
     @classmethod
     def create(cls, user, repository, permission):
         n = cls()
         n.user = user
         n.repository = repository
         n.permission = permission
         Session.add(n)
         return n
 
     def __repr__(self):
         return '<user:%s => %s >' % (self.user, self.repository)
 
 
 class UserToPerm(Base, BaseModel):
     __tablename__ = 'user_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'permission_id'),
         {'extend_existing': True}
     )
     user_to_perm_id = Column("user_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User')
     permission = relationship('Permission', lazy='joined')
 
 
 class UsersGroupRepoToPerm(Base, BaseModel):
     __tablename__ = 'users_group_repo_to_perm'
     __table_args__ = (
         UniqueConstraint('repository_id', 'users_group_id', 'permission_id'),
         {'extend_existing': True}
     )
     users_group_to_perm_id = Column("users_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
 
     users_group = relationship('UsersGroup')
     permission = relationship('Permission')
     repository = relationship('Repository')
 
     @classmethod
     def create(cls, users_group, repository, permission):
         n = cls()
         n.users_group = users_group
         n.repository = repository
         n.permission = permission
         Session.add(n)
         return n
 
     def __repr__(self):
         return '<userGroup:%s => %s >' % (self.users_group, self.repository)
 
 
 class UsersGroupToPerm(Base, BaseModel):
     __tablename__ = 'users_group_to_perm'
     __table_args__ = (
         UniqueConstraint('users_group_id', 'permission_id',),
         {'extend_existing': True}
     )
     users_group_to_perm_id = Column("users_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
 
     users_group = relationship('UsersGroup')
     permission = relationship('Permission')
 
 
 class UserRepoGroupToPerm(Base, BaseModel):
     __tablename__ = 'user_repo_group_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'group_id', 'permission_id'),
         {'extend_existing': True}
     )
 
     group_to_perm_id = Column("group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User')
     group = relationship('RepoGroup')
     permission = relationship('Permission')
 
 
 class UsersGroupRepoGroupToPerm(Base, BaseModel):
     __tablename__ = 'users_group_repo_group_to_perm'
     __table_args__ = (
         UniqueConstraint('users_group_id', 'group_id'),
         {'extend_existing': True}
     )
 
     users_group_repo_group_to_perm_id = Column("users_group_repo_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
 
     users_group = relationship('UsersGroup')
     permission = relationship('Permission')
     group = relationship('RepoGroup')
 
 
 class Statistics(Base, BaseModel):
     __tablename__ = 'statistics'
     __table_args__ = (UniqueConstraint('repository_id'), {'extend_existing': True})
     stat_id = Column("stat_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=True, default=None)
     stat_on_revision = Column("stat_on_revision", Integer(), nullable=False)
     commit_activity = Column("commit_activity", LargeBinary(1000000), nullable=False)#JSON data
     commit_activity_combined = Column("commit_activity_combined", LargeBinary(), nullable=False)#JSON data
     languages = Column("languages", LargeBinary(1000000), nullable=False)#JSON data
 
     repository = relationship('Repository', single_parent=True)
 
 
 class UserFollowing(Base, BaseModel):
     __tablename__ = 'user_followings'
     __table_args__ = (
         UniqueConstraint('user_id', 'follows_repository_id'),
         UniqueConstraint('user_id', 'follows_user_id'),
         {'extend_existing': True}
     )
 
     user_following_id = Column("user_following_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     follows_repo_id = Column("follows_repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=True, unique=None, default=None)
     follows_user_id = Column("follows_user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
     follows_from = Column('follows_from', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)
 
     user = relationship('User', primaryjoin='User.user_id==UserFollowing.user_id')
 
     follows_user = relationship('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')
     follows_repository = relationship('Repository', order_by='Repository.repo_name')
 
     @classmethod
     def get_repo_followers(cls, repo_id):
         return cls.query().filter(cls.follows_repo_id == repo_id)
 
 
 class CacheInvalidation(Base, BaseModel):
     __tablename__ = 'cache_invalidation'
     __table_args__ = (UniqueConstraint('cache_key'), {'extend_existing': True})
     cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     cache_key = Column("cache_key", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     cache_args = Column("cache_args", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False)
 
     def __init__(self, cache_key, cache_args=''):
         self.cache_key = cache_key
         self.cache_args = cache_args
         self.cache_active = False
 
     def __repr__(self):
         return "<%s('%s:%s')>" % (self.__class__.__name__,
                                   self.cache_id, self.cache_key)
 
     @classmethod
     def invalidate(cls, key):
         """
         Returns Invalidation object if this given key should be invalidated
         None otherwise. `cache_active = False` means that this cache
         state is not valid and needs to be invalidated
 
         :param key:
         """
         return cls.query()\
                 .filter(CacheInvalidation.cache_key == key)\
                 .filter(CacheInvalidation.cache_active == False)\
                 .scalar()
 
     @classmethod
     def set_invalidate(cls, key):
         """
         Mark this Cache key for invalidation
 
         :param key:
         """
 
         log.debug('marking %s for invalidation' % key)
         inv_obj = Session.query(cls)\
             .filter(cls.cache_key == key).scalar()
         if inv_obj:
             inv_obj.cache_active = False
         else:
             log.debug('cache key not found in invalidation db -> creating one')
             inv_obj = CacheInvalidation(key)
 
         try:
             Session.add(inv_obj)
             Session.commit()
         except Exception:
             log.error(traceback.format_exc())
             Session.rollback()
 
     @classmethod
     def set_valid(cls, key):
         """
         Mark this cache key as active and currently cached
 
         :param key:
         """
         inv_obj = CacheInvalidation.query()\
             .filter(CacheInvalidation.cache_key == key).scalar()
         inv_obj.cache_active = True
         Session.add(inv_obj)
         Session.commit()
 
 
 class ChangesetComment(Base, BaseModel):
     __tablename__ = 'changeset_comments'
     __table_args__ = ({'extend_existing': True},)
     comment_id = Column('comment_id', Integer(), nullable=False, primary_key=True)
     repo_id = Column('repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     revision = Column('revision', String(40), nullable=False)
     line_no = Column('line_no', Unicode(10), nullable=True)
     f_path = Column('f_path', Unicode(1000), nullable=True)
     user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     text = Column('text', Unicode(25000), nullable=False)
     modified_at = Column('modified_at', DateTime(), nullable=False, default=datetime.datetime.now)
 
     author = relationship('User', lazy='joined')
     repo = relationship('Repository')
 
     @classmethod
     def get_users(cls, revision):
         """
         Returns user associated with this changesetComment. ie those
         who actually commented
 
         :param cls:
         :param revision:
         """
         return Session.query(User)\
                 .filter(cls.revision == revision)\
                 .join(ChangesetComment.author).all()
 
 
 class Notification(Base, BaseModel):
     __tablename__ = 'notifications'
     __table_args__ = ({'extend_existing': True},)
 
     TYPE_CHANGESET_COMMENT = u'cs_comment'
     TYPE_MESSAGE = u'message'
     TYPE_MENTION = u'mention'
     TYPE_REGISTRATION = u'registration'
 
     notification_id = Column('notification_id', Integer(), nullable=False, primary_key=True)
     subject = Column('subject', Unicode(512), nullable=True)
     body = Column('body', Unicode(50000), nullable=True)
     created_by = Column("created_by", Integer(), ForeignKey('users.user_id'), nullable=True)
     created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     type_ = Column('type', Unicode(256))
 
     created_by_user = relationship('User')
     notifications_to_users = relationship('UserNotification', lazy='joined',
                                           cascade="all, delete, delete-orphan")
 
     @property
     def recipients(self):
         return [x.user for x in UserNotification.query()\
                 .filter(UserNotification.notification == self).all()]
 
     @classmethod
     def create(cls, created_by, subject, body, recipients, type_=None):
         if type_ is None:
             type_ = Notification.TYPE_MESSAGE
 
         notification = cls()
         notification.created_by_user = created_by
         notification.subject = subject
         notification.body = body
         notification.type_ = type_
         notification.created_on = datetime.datetime.now()
 
         for u in recipients:
             assoc = UserNotification()
             assoc.notification = notification
             u.notifications.append(assoc)
         Session.add(notification)
         return notification
 
     @property
     def description(self):
         from rhodecode.model.notification import NotificationModel
         return NotificationModel().make_description(self)
 
 
 class UserNotification(Base, BaseModel):
     __tablename__ = 'user_to_notification'
     __table_args__ = (
         UniqueConstraint('user_id', 'notification_id'),
         {'extend_existing': True}
     )
     user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), primary_key=True)
     notification_id = Column("notification_id", Integer(), ForeignKey('notifications.notification_id'), primary_key=True)
     read = Column('read', Boolean, default=False)
     sent_on = Column('sent_on', DateTime(timezone=False), nullable=True, unique=None)
 
     user = relationship('User', lazy="joined")
     notification = relationship('Notification', lazy="joined",
                                 order_by=lambda: Notification.created_on.desc(),)
 
     def mark_as_read(self):
         self.read = True
         Session.add(self)
 
 
 class DbMigrateVersion(Base, BaseModel):
     __tablename__ = 'db_migrate_version'
     __table_args__ = {'extend_existing': True}
     repository_id = Column('repository_id', String(250), primary_key=True)
     repository_path = Column('repository_path', Text)
     version = Column('version', Integer)